Facebook
From Diminutive Ostrich, 3 Years ago, written in Plain Text.
Embed
Download Paste or View Raw
Hits: 132
  1.  
  2.  
  3. [2020-09-29T12:01:36,747][INFO ][o.e.c.c.Coordinator      ] [serverra2_warm.sit.comp.state] master node [{serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}] failed, restarting discovery
  4. org.elasticsearch.ElasticsearchException: node [{serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}] failed [9] consecutive checks
  5.         at org.elasticsearch.cluster.coordination.LeaderChecker$CheckScheduler$1.handleException(LeaderChecker.java:277) ~[elasticsearch-7.7.0.jar:7.7.0]
  6.         at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1139) ~[elasticsearch-7.7.0.jar:7.7.0]
  7.         at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1139) ~[elasticsearch-7.7.0.jar:7.7.0]
  8.         at org.elasticsearch.transport.InboundHandler.lambda$handleException$2(InboundHandler.java:244) ~[elasticsearch-7.7.0.jar:7.7.0]
  9.         at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.7.0.jar:7.7.0]
  10.         at org.elasticsearch.transport.InboundHandler.handleException(InboundHandler.java:242) ~[elasticsearch-7.7.0.jar:7.7.0]
  11.         at org.elasticsearch.transport.InboundHandler.handlerResponseError(InboundHandler.java:234) ~[elasticsearch-7.7.0.jar:7.7.0]
  12.         at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:137) ~[elasticsearch-7.7.0.jar:7.7.0]
  13.         at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:103) ~[elasticsearch-7.7.0.jar:7.7.0]
  14.         at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:676) ~[elasticsearch-7.7.0.jar:7.7.0]
  15.         at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:62) ~[?:?]
  16.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  17.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  18.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  19.         at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321) ~[?:?]
  20.         at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295) ~[?:?]
  21.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  22.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  23.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  24.         at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:227) ~[?:?]
  25.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  26.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  27.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  28.         at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1470) ~[?:?]
  29.         at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1219) ~[?:?]
  30.         at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1266) ~[?:?]
  31.         at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:498) ~[?:?]
  32.         at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:437) ~[?:?]
  33.         at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
  34.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  35.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  36.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  37.         at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
  38.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  39.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  40.         at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
  41.         at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?]
  42.         at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) ~[?:?]
  43.         at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]
  44.         at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]
  45.         at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
  46.         at io.netty.util.concurrent.SingleTstateeadEventExecutor$4.run(SingleTstateeadEventExecutor.java:989) ~[?:?]
  47.         at io.netty.util.internal.TstateeadExecutorMap$2.run(TstateeadExecutorMap.java:74) ~[?:?]
  48.         at java.lang.Tstateead.run(Tstateead.java:832) [?:?]
  49. Caused by: org.elasticsearch.transport.RemoteTransportException: [serverra2.sit.comp.state][10.100.24.231:9300][internal:coordination/fault_detection/leader_check]
  50. Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: rejecting leader check since [{serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=warm, transform.node=true}] has been removed from the cluster
  51.         at org.elasticsearch.cluster.coordination.LeaderChecker.handleLeaderCheck(LeaderChecker.java:180) ~[elasticsearch-7.7.0.jar:7.7.0]
  52.         at org.elasticsearch.cluster.coordination.LeaderChecker.lambda$new$0(LeaderChecker.java:106) ~[elasticsearch-7.7.0.jar:7.7.0]
  53.         at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) ~[?:?]
  54.         at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.7.0.jar:7.7.0]
  55.         at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.7.0.jar:7.7.0]
  56.         at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) ~[?:?]
  57.         at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) ~[elasticsearch-7.7.0.jar:7.7.0]
  58.         at org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) ~[?:?]
  59.         at org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) ~[?:?]
  60.         at org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) ~[?:?]
  61.         at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) ~[elasticsearch-7.7.0.jar:7.7.0]
  62.         at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) ~[?:?]
  63.         at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) ~[?:?]
  64.         at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) ~[?:?]
  65.         at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) ~[?:?]
  66.         at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) ~[?:?]
  67.         at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) ~[?:?]
  68.         at org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) ~[?:?]
  69.         at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) ~[?:?]
  70.         at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.7.0.jar:7.7.0]
  71.         at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) ~[elasticsearch-7.7.0.jar:7.7.0]
  72.         at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.7.0.jar:7.7.0]
  73.         at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.7.0.jar:7.7.0]
  74.         at org.elasticsearch.transport.InboundHandler.handleRequest(InboundHandler.java:186) ~[elasticsearch-7.7.0.jar:7.7.0]
  75.         at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:119) ~[elasticsearch-7.7.0.jar:7.7.0]
  76.         at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:103) ~[elasticsearch-7.7.0.jar:7.7.0]
  77.         at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:676) ~[elasticsearch-7.7.0.jar:7.7.0]
  78.         at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:62) ~[?:?]
  79.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  80.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  81.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  82.         at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321) ~[?:?]
  83.         at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295) ~[?:?]
  84.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  85.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  86.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  87.         at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:227) ~[?:?]
  88.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  89.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  90.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  91.         at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1470) ~[?:?]
  92.         at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1219) ~[?:?]
  93.         at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1266) ~[?:?]
  94.         at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:498) ~[?:?]
  95.         at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:437) ~[?:?]
  96.         at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
  97.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  98.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  99.         at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
  100.         at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
  101.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
  102.         at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
  103.         at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
  104.         at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?]
  105.         at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) ~[?:?]
  106.         at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]
  107.         at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]
  108.         at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
  109.         at io.netty.util.concurrent.SingleTstateeadEventExecutor$4.run(SingleTstateeadEventExecutor.java:989) ~[?:?]
  110.         at io.netty.util.internal.TstateeadExecutorMap$2.run(TstateeadExecutorMap.java:74) ~[?:?]
  111.         at java.lang.Tstateead.run(Tstateead.java:832) ~[?:?]
  112. [2020-09-29T12:02:37,412][INFO ][o.e.c.c.JoinHelper       ] [serverra2_warm.sit.comp.state] failed to join {serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true} with JoinRequest{sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, minimumTerm=68, optionalJoin=Optional[Join{term=68, lastAcceptedTerm=67, lastAcceptedVersion=412816, sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, targetNode={serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}}]}
  113. org.elasticsearch.transport.ReceiveTimeoutTransportException: [serverra2.sit.comp.state][10.100.24.231:9300][internal:cluster/coordination/join] request_id [9658363] timed out after [59876ms]
  114.         at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:1041) [elasticsearch-7.7.0.jar:7.7.0]
  115.         at org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633) [elasticsearch-7.7.0.jar:7.7.0]
  116.         at java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) [?:?]
  117.         at java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) [?:?]
  118.         at java.lang.Tstateead.run(Tstateead.java:832) [?:?]
  119. [2020-09-29T12:02:37,413][INFO ][o.e.c.c.JoinHelper       ] [serverra2_warm.sit.comp.state] failed to join {serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true} with JoinRequest{sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, minimumTerm=68, optionalJoin=Optional[Join{term=68, lastAcceptedTerm=67, lastAcceptedVersion=412816, sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, targetNode={serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}}]}
  120. org.elasticsearch.transport.ReceiveTimeoutTransportException: [serverra2.sit.comp.state][10.100.24.231:9300][internal:cluster/coordination/join] request_id [9658363] timed out after [59876ms]
  121.         at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:1041) [elasticsearch-7.7.0.jar:7.7.0]
  122.         at org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633) [elasticsearch-7.7.0.jar:7.7.0]
  123.         at java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) [?:?]
  124.         at java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) [?:?]
  125.         at java.lang.Tstateead.run(Tstateead.java:832) [?:?]
  126. [2020-09-29T12:03:39,392][WARN ][o.e.t.TransportService   ] [serverra2_warm.sit.comp.state] Received response for a request that has timed out, sent [121910ms] ago, timed out [62034ms] ago, action [internal:cluster/coordination/join], node [{serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}], id [9658363]
  127. ^C
  128.  
  129.