::: {serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=warm, transform.node=true}
   Hot tstateeads at 2020-09-29T10:00:14.288Z, interval=500ms, busiestTstateeads=99999, ignoreIdleTstateeads=true:

    5.3% (26.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1072]'
     10/10 snapshots sharing following 56 elements
       java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252)
       app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272)
       app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242)
       app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141)
       app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
       app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
       app//org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.<init>(Lucene50CompoundReader.java:79)
       app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.getCompoundReader(Lucene50CompoundFormat.java:70)
       app//org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:101)
       app//org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
       app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
       app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
       app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
       app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
       app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
       app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92)
       app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644)
       app//org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:260)
       app//org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:200)
       app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
       app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625)
       app//org.elasticsearch.index.shard.IndexShard.openEngineAndSkipTranslogRecovery(IndexShard.java:1603)
       app//org.elasticsearch.indices.recovery.RecoveryTarget.lambda$prepareForTranslogOperations$0(RecoveryTarget.java:287)
       app//org.elasticsearch.indices.recovery.RecoveryTarget$$Lambda$4887/0x00007f5b9c8424b0.get(Unknown Source)
       app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325)
       app//org.elasticsearch.indices.recovery.RecoveryTarget.prepareForTranslogOperations(RecoveryTarget.java:285)
       app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$PrepareForTranslogOperationsRequestHandler.messageReceived(PeerRecoveryTargetService.java:383)
       app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$PrepareForTranslogOperationsRequestHandler.messageReceived(PeerRecoveryTargetService.java:377)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385)
       org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263)
       org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313)
       app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
       app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
       app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.1% (411.7micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][scheduler][T#1]'
     7/10 snapshots sharing following 2 elements
       java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.0% (45.9micros out of 500ms) cpu usage by tstateead 'ticker-schedule-trigger-engine'
     10/10 snapshots sharing following 2 elements
       java.base@14/java.lang.Tstateead.sleep(Native Method)
       org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:193)

    0.0% (40.2micros out of 500ms) cpu usage by tstateead 'Connection evictor'
     10/10 snapshots sharing following 3 elements
       java.base@14/java.lang.Tstateead.sleep(Native Method)
       org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.0% (0s out of 500ms) cpu usage by tstateead 'Common-Cleaner'
     10/10 snapshots sharing following 5 elements
       java.base@14/java.lang.Object.wait(Native Method)
       java.base@14/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
       java.base@14/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
       java.base@14/jdk.internal.misc.InnocuousTstateead.run(InnocuousTstateead.java:134)

    0.0% (0s out of 500ms) cpu usage by tstateead 'Notification Tstateead'
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot

    0.0% (0s out of 500ms) cpu usage by tstateead 'process reaper'
     10/10 snapshots sharing following 5 elements
       java.base@14/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
       java.base@14/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:138)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][clusterApplierService#updateTask][T#1]'
     10/10 snapshots sharing following 15 elements
       app//org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:217)
       app//org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateAppliers$5(ClusterApplierService.java:517)
       app//org.elasticsearch.cluster.service.ClusterApplierService$$Lambda$4256/0x00007f5d795180b0.accept(Unknown Source)
       java.base@14/java.lang.Iterable.forEach(Iterable.java:75)
       app//org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:514)
       app//org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:485)
       app//org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:432)
       app//org.elasticsearch.cluster.service.ClusterApplierService.access$100(ClusterApplierService.java:73)
       app//org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:176)
       app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633)
       app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsTstateeadPoolExecutor.java:252)
       app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsTstateeadPoolExecutor.java:215)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.0% (0s out of 500ms) cpu usage by tstateead 'ml-cpp-log-tail-tstateead'
     10/10 snapshots sharing following 6 elements
       java.base@14/java.io.FileInputStream.readBytes(Native Method)
       java.base@14/java.io.FileInputStream.read(FileInputStream.java:250)
       org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:105)
       org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInTstateead$0(NativeController.java:72)
       org.elasticsearch.xpack.ml.process.NativeController$$Lambda$2548/0x00007f5ee420a440.run(Unknown Source)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.0% (0s out of 500ms) cpu usage by tstateead 'DestroyJavaVM'
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot
     unique snapshot

    0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][management][T#1]'
     2/10 snapshots sharing following 29 elements
       app//org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:388)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385)
       org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source)
       app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321)
       org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263)
       org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174)
       org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120)
       org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313)
       app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
       app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
       app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692)
       app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
     2/10 snapshots sharing following 9 elements
       java.base@14/jdk.internal.misc.Unsafe.park(Native Method)
       java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
       java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740)
       java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
       java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[keepAlive/7.7.0]'
     10/10 snapshots sharing following 7 elements
       java.base@14/jdk.internal.misc.Unsafe.park(Native Method)
       java.base@14/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
       java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer.acquire(AbstractQueuedSyncstateonizer.java:714)
       java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer.acquireSharedInterruptibly(AbstractQueuedSyncstateonizer.java:1046)
       java.base@14/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
       app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:89)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)

    0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1001]'
     10/10 snapshots sharing following 20 elements
       app//org.elasticsearch.index.shard.IndexShard.close(IndexShard.java:1336)
       app//org.elasticsearch.index.IndexService.closeShard(IndexService.java:520)
       app//org.elasticsearch.index.IndexService.removeShard(IndexService.java:503)
       app//org.elasticsearch.indices.cluster.IndicesClusterStateService.failAndRemoveShard(IndicesClusterStateService.java:714)
       app//org.elasticsearch.indices.cluster.IndicesClusterStateService.handleRecoveryFailure(IndicesClusterStateService.java:704)
       app//org.elasticsearch.indices.cluster.IndicesClusterStateService$RecoveryListener.onRecoveryFailure(IndicesClusterStateService.java:698)
       app//org.elasticsearch.indices.recovery.RecoveryTarget.notifyListener(RecoveryTarget.java:238)
       app//org.elasticsearch.indices.recovery.RecoveryTarget.fail(RecoveryTarget.java:225)
       app//org.elasticsearch.indices.recovery.RecoveriesCollection.failRecovery(RecoveriesCollection.java:177)
       app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService.lambda$doRecovery$2(PeerRecoveryTargetService.java:247)
       app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$$Lambda$4777/0x00007e4847552908.accept(Unknown Source)
       app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$1.handleException(PeerRecoveryTargetService.java:292)
       app//org.elasticsearch.transport.PlainTransportFuture.handleException(PlainTransportFuture.java:97)
       app//org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1139)
       app//org.elasticsearch.transport.InboundHandler.lambda$handleException$2(InboundHandler.java:244)
       app//org.elasticsearch.transport.InboundHandler$$Lambda$4193/0x00007f5ead3e84b0.run(Unknown Source)
       app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
       java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
       java.base@14/java.lang.Tstateead.run(Tstateead.java:832)