[2020-09-29T12:01:36,747][INFO ][o.e.c.c.Coordinator      ] [serverra2_warm.sit.comp.state] master node [{serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}] failed, restarting discovery
org.elasticsearch.ElasticsearchException: node [{serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}] failed [9] consecutive checks
        at org.elasticsearch.cluster.coordination.LeaderChecker$CheckScheduler$1.handleException(LeaderChecker.java:277) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1139) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1139) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.lambda$handleException$2(InboundHandler.java:244) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.handleException(InboundHandler.java:242) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.handlerResponseError(InboundHandler.java:234) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:137) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:103) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:676) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:62) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:227) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1470) ~[?:?]
        at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1219) ~[?:?]
        at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1266) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:498) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:437) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
        at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
        at io.netty.util.concurrent.SingleTstateeadEventExecutor$4.run(SingleTstateeadEventExecutor.java:989) ~[?:?]
        at io.netty.util.internal.TstateeadExecutorMap$2.run(TstateeadExecutorMap.java:74) ~[?:?]
        at java.lang.Tstateead.run(Tstateead.java:832) [?:?]
Caused by: org.elasticsearch.transport.RemoteTransportException: [serverra2.sit.comp.state][10.100.24.231:9300][internal:coordination/fault_detection/leader_check]
Caused by: org.elasticsearch.cluster.coordination.CoordinationStateRejectedException: rejecting leader check since [{serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=warm, transform.node=true}] has been removed from the cluster
        at org.elasticsearch.cluster.coordination.LeaderChecker.handleLeaderCheck(LeaderChecker.java:180) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.cluster.coordination.LeaderChecker.lambda$new$0(LeaderChecker.java:106) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) ~[?:?]
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) ~[?:?]
        at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) ~[?:?]
        at org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) ~[?:?]
        at org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) ~[?:?]
        at org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) ~[?:?]
        at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) ~[?:?]
        at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) ~[?:?]
        at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) ~[?:?]
        at org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) ~[?:?]
        at org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) ~[?:?]
        at org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) ~[?:?]
        at org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) ~[?:?]
        at org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.handleRequest(InboundHandler.java:186) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.messageReceived(InboundHandler.java:119) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.InboundHandler.inboundMessage(InboundHandler.java:103) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.TcpTransport.inboundMessage(TcpTransport.java:676) ~[elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.transport.netty4.Netty4MessageChannelHandler.channelRead(Netty4MessageChannelHandler.java:62) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.fireChannelRead(ByteToMessageDecoder.java:321) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:295) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.handler.logging.LoggingHandler.channelRead(LoggingHandler.java:227) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.handler.ssl.SslHandler.unwrap(SslHandler.java:1470) ~[?:?]
        at io.netty.handler.ssl.SslHandler.decodeJdkCompatible(SslHandler.java:1219) ~[?:?]
        at io.netty.handler.ssl.SslHandler.decode(SslHandler.java:1266) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.decodeRemovalReentryProtection(ByteToMessageDecoder.java:498) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.callDecode(ByteToMessageDecoder.java:437) ~[?:?]
        at io.netty.handler.codec.ByteToMessageDecoder.channelRead(ByteToMessageDecoder.java:276) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.fireChannelRead(AbstractChannelHandlerContext.java:355) ~[?:?]
        at io.netty.channel.DefaultChannelPipeline$HeadContext.channelRead(DefaultChannelPipeline.java:1410) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:377) ~[?:?]
        at io.netty.channel.AbstractChannelHandlerContext.invokeChannelRead(AbstractChannelHandlerContext.java:363) ~[?:?]
        at io.netty.channel.DefaultChannelPipeline.fireChannelRead(DefaultChannelPipeline.java:919) ~[?:?]
        at io.netty.channel.nio.AbstractNioByteChannel$NioByteUnsafe.read(AbstractNioByteChannel.java:163) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.processSelectedKey(NioEventLoop.java:714) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeysPlain(NioEventLoop.java:615) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.processSelectedKeys(NioEventLoop.java:578) ~[?:?]
        at io.netty.channel.nio.NioEventLoop.run(NioEventLoop.java:493) ~[?:?]
        at io.netty.util.concurrent.SingleTstateeadEventExecutor$4.run(SingleTstateeadEventExecutor.java:989) ~[?:?]
        at io.netty.util.internal.TstateeadExecutorMap$2.run(TstateeadExecutorMap.java:74) ~[?:?]
        at java.lang.Tstateead.run(Tstateead.java:832) ~[?:?]
[2020-09-29T12:02:37,412][INFO ][o.e.c.c.JoinHelper       ] [serverra2_warm.sit.comp.state] failed to join {serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true} with JoinRequest{sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, minimumTerm=68, optionalJoin=Optional[Join{term=68, lastAcceptedTerm=67, lastAcceptedVersion=412816, sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, targetNode={serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}}]}
org.elasticsearch.transport.ReceiveTimeoutTransportException: [serverra2.sit.comp.state][10.100.24.231:9300][internal:cluster/coordination/join] request_id [9658363] timed out after [59876ms]
        at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:1041) [elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633) [elasticsearch-7.7.0.jar:7.7.0]
        at java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) [?:?]
        at java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) [?:?]
        at java.lang.Tstateead.run(Tstateead.java:832) [?:?]
[2020-09-29T12:02:37,413][INFO ][o.e.c.c.JoinHelper       ] [serverra2_warm.sit.comp.state] failed to join {serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true} with JoinRequest{sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, minimumTerm=68, optionalJoin=Optional[Join{term=68, lastAcceptedTerm=67, lastAcceptedVersion=412816, sourceNode={serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, xpack.installed=true, data=warm, transform.node=true, ml.max_open_jobs=20}, targetNode={serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}}]}
org.elasticsearch.transport.ReceiveTimeoutTransportException: [serverra2.sit.comp.state][10.100.24.231:9300][internal:cluster/coordination/join] request_id [9658363] timed out after [59876ms]
        at org.elasticsearch.transport.TransportService$TimeoutHandler.run(TransportService.java:1041) [elasticsearch-7.7.0.jar:7.7.0]
        at org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633) [elasticsearch-7.7.0.jar:7.7.0]
        at java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) [?:?]
        at java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) [?:?]
        at java.lang.Tstateead.run(Tstateead.java:832) [?:?]
[2020-09-29T12:03:39,392][WARN ][o.e.t.TransportService   ] [serverra2_warm.sit.comp.state] Received response for a request that has timed out, sent [121910ms] ago, timed out [62034ms] ago, action [internal:cluster/coordination/join], node [{serverra2.sit.comp.state}{wHODtVp4QJmjUVaO86eUPw}{BU0qh0kUR6O2YzrQ6Onxdg}{10.100.24.231}{10.100.24.231:9300}{dilmrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=hot, transform.node=true}], id [9658363]
^C