1. ::: {serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=warm, transform.node=true}
2.    Hot tstateeads at 2020-09-29T10:00:14.288Z, interval=500ms, busiestTstateeads=99999, ignoreIdleTstateeads=true:
3.  
4.     5.3% (26.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1072]'
5.      10/10 snapshots sharing following 56 elements
6.        java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252)
7.        app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272)
8.        app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242)
9.        app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141)
10.        app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
11.        app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100)
12.        app//org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.<init>(Lucene50CompoundReader.java:79)
13.        app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.getCompoundReader(Lucene50CompoundFormat.java:70)
14.        app//org.apache.lucene.index.SegmentCoreReaders.<init>(SegmentCoreReaders.java:101)
15.        app//org.apache.lucene.index.SegmentReader.<init>(SegmentReader.java:84)
16.        app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177)
17.        app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219)
18.        app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109)
19.        app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526)
20.        app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116)
21.        app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92)
22.        app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644)
23.        app//org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:260)
24.        app//org.elasticsearch.index.engine.InternalEngine.<init>(InternalEngine.java:200)
25.        app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25)
26.        app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625)
27.        app//org.elasticsearch.index.shard.IndexShard.openEngineAndSkipTranslogRecovery(IndexShard.java:1603)
28.        app//org.elasticsearch.indices.recovery.RecoveryTarget.lambda$prepareForTranslogOperations$0(RecoveryTarget.java:287)
29.        app//org.elasticsearch.indices.recovery.RecoveryTarget$$Lambda$4887/0x00007f5b9c8424b0.get(Unknown Source)
30.        app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325)
31.        app//org.elasticsearch.indices.recovery.RecoveryTarget.prepareForTranslogOperations(RecoveryTarget.java:285)
32.        app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$PrepareForTranslogOperationsRequestHandler.messageReceived(PeerRecoveryTargetService.java:383)
33.        app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$PrepareForTranslogOperationsRequestHandler.messageReceived(PeerRecoveryTargetService.java:377)
34.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257)
35.        app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
36.        app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225)
37.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306)
38.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source)
39.        app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
40.        org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385)
41.        org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190)
42.        org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129)
43.        org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source)
44.        app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
45.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324)
46.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source)
47.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386)
48.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source)
49.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397)
50.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321)
51.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263)
52.        org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174)
53.        org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120)
54.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313)
55.        app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
56.        app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
57.        app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692)
58.        app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
59.        java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
60.        java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
61.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
62.  
63.     0.1% (411.7micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][scheduler][T#1]'
64.      7/10 snapshots sharing following 2 elements
65.        java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
66.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
67.  
68.     0.0% (45.9micros out of 500ms) cpu usage by tstateead 'ticker-schedule-trigger-engine'
69.      10/10 snapshots sharing following 2 elements
70.        java.base@14/java.lang.Tstateead.sleep(Native Method)
71.        org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:193)
72.  
73.     0.0% (40.2micros out of 500ms) cpu usage by tstateead 'Connection evictor'
74.      10/10 snapshots sharing following 3 elements
75.        java.base@14/java.lang.Tstateead.sleep(Native Method)
76.        org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66)
77.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
78.  
79.     0.0% (0s out of 500ms) cpu usage by tstateead 'Common-Cleaner'
80.      10/10 snapshots sharing following 5 elements
81.        java.base@14/java.lang.Object.wait(Native Method)
82.        java.base@14/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155)
83.        java.base@14/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148)
84.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
85.        java.base@14/jdk.internal.misc.InnocuousTstateead.run(InnocuousTstateead.java:134)
86.  
87.     0.0% (0s out of 500ms) cpu usage by tstateead 'Notification Tstateead'
88.      unique snapshot
89.      unique snapshot
90.      unique snapshot
91.      unique snapshot
92.      unique snapshot
93.      unique snapshot
94.      unique snapshot
95.      unique snapshot
96.      unique snapshot
97.      unique snapshot
98.  
99.     0.0% (0s out of 500ms) cpu usage by tstateead 'process reaper'
100.      10/10 snapshots sharing following 5 elements
101.        java.base@14/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method)
102.        java.base@14/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:138)
103.        java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
104.        java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
105.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
106.  
107.     0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][clusterApplierService#updateTask][T#1]'
108.      10/10 snapshots sharing following 15 elements
109.        app//org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:217)
110.        app//org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateAppliers$5(ClusterApplierService.java:517)
111.        app//org.elasticsearch.cluster.service.ClusterApplierService$$Lambda$4256/0x00007f5d795180b0.accept(Unknown Source)
112.        java.base@14/java.lang.Iterable.forEach(Iterable.java:75)
113.        app//org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:514)
114.        app//org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:485)
115.        app//org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:432)
116.        app//org.elasticsearch.cluster.service.ClusterApplierService.access$100(ClusterApplierService.java:73)
117.        app//org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:176)
118.        app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633)
119.        app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsTstateeadPoolExecutor.java:252)
120.        app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsTstateeadPoolExecutor.java:215)
121.        java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
122.        java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
123.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
124.  
125.     0.0% (0s out of 500ms) cpu usage by tstateead 'ml-cpp-log-tail-tstateead'
126.      10/10 snapshots sharing following 6 elements
127.        java.base@14/java.io.FileInputStream.readBytes(Native Method)
128.        java.base@14/java.io.FileInputStream.read(FileInputStream.java:250)
129.        org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:105)
130.        org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInTstateead$0(NativeController.java:72)
131.        org.elasticsearch.xpack.ml.process.NativeController$$Lambda$2548/0x00007f5ee420a440.run(Unknown Source)
132.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
133.  
134.     0.0% (0s out of 500ms) cpu usage by tstateead 'DestroyJavaVM'
135.      unique snapshot
136.      unique snapshot
137.      unique snapshot
138.      unique snapshot
139.      unique snapshot
140.      unique snapshot
141.      unique snapshot
142.      unique snapshot
143.      unique snapshot
144.      unique snapshot
145.  
146.     0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][management][T#1]'
147.      2/10 snapshots sharing following 29 elements
148.        app//org.elasticsearch.action.support.broadcast.node.TransportBroadcastByNodeAction$BroadcastByNodeTransportRequestHandler.messageReceived(TransportBroadcastByNodeAction.java:388)
149.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257)
150.        app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
151.        app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225)
152.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306)
153.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source)
154.        app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
155.        org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385)
156.        org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190)
157.        org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129)
158.        org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source)
159.        app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63)
160.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324)
161.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source)
162.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386)
163.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source)
164.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397)
165.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321)
166.        org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263)
167.        org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174)
168.        org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120)
169.        org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313)
170.        app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63)
171.        app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264)
172.        app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692)
173.        app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37)
174.        java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
175.        java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
176.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
177.      2/10 snapshots sharing following 9 elements
178.        java.base@14/jdk.internal.misc.Unsafe.park(Native Method)
179.        java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252)
180.        java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740)
181.        java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684)
182.        java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374)
183.        java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055)
184.        java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116)
185.        java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
186.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
187.  
188.     0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[keepAlive/7.7.0]'
189.      10/10 snapshots sharing following 7 elements
190.        java.base@14/jdk.internal.misc.Unsafe.park(Native Method)
191.        java.base@14/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211)
192.        java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer.acquire(AbstractQueuedSyncstateonizer.java:714)
193.        java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer.acquireSharedInterruptibly(AbstractQueuedSyncstateonizer.java:1046)
194.        java.base@14/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232)
195.        app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:89)
196.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
197.  
198.     0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1001]'
199.      10/10 snapshots sharing following 20 elements
200.        app//org.elasticsearch.index.shard.IndexShard.close(IndexShard.java:1336)
201.        app//org.elasticsearch.index.IndexService.closeShard(IndexService.java:520)
202.        app//org.elasticsearch.index.IndexService.removeShard(IndexService.java:503)
203.        app//org.elasticsearch.indices.cluster.IndicesClusterStateService.failAndRemoveShard(IndicesClusterStateService.java:714)
204.        app//org.elasticsearch.indices.cluster.IndicesClusterStateService.handleRecoveryFailure(IndicesClusterStateService.java:704)
205.        app//org.elasticsearch.indices.cluster.IndicesClusterStateService$RecoveryListener.onRecoveryFailure(IndicesClusterStateService.java:698)
206.        app//org.elasticsearch.indices.recovery.RecoveryTarget.notifyListener(RecoveryTarget.java:238)
207.        app//org.elasticsearch.indices.recovery.RecoveryTarget.fail(RecoveryTarget.java:225)
208.        app//org.elasticsearch.indices.recovery.RecoveriesCollection.failRecovery(RecoveriesCollection.java:177)
209.        app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService.lambda$doRecovery$2(PeerRecoveryTargetService.java:247)
210.        app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$$Lambda$4777/0x00007e4847552908.accept(Unknown Source)
211.        app//org.elasticsearch.indices.recovery.PeerRecoveryTargetService$1.handleException(PeerRecoveryTargetService.java:292)
212.        app//org.elasticsearch.transport.PlainTransportFuture.handleException(PlainTransportFuture.java:97)
213.        app//org.elasticsearch.transport.TransportService$ContextRestoreResponseHandler.handleException(TransportService.java:1139)
214.        app//org.elasticsearch.transport.InboundHandler.lambda$handleException$2(InboundHandler.java:244)
215.        app//org.elasticsearch.transport.InboundHandler$$Lambda$4193/0x00007f5ead3e84b0.run(Unknown Source)
216.        app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633)
217.        java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130)
218.        java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630)
219.        java.base@14/java.lang.Tstateead.run(Tstateead.java:832)
220.  
221.  
222.