::: {serverra2_warm.sit.comp.state}{KiQQGwdoTgWwHYzMItyhXQ}{OjCMtVthSlqJk-TmL5UH8A}{10.100.24.231}{10.100.24.231:9301}{dlrt}{rack_id=rack_one, ml.machine_memory=269645852672, ml.max_open_jobs=20, xpack.installed=true, data=warm, transform.node=true} Hot tstateeads at 2020-09-29T11:28:15.220Z, interval=500ms, busiestTstateeads=99999, ignoreIdleTstateeads=true: 11.4% (57.1ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4625]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3.5% (17.5ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1042]' 10/10 snapshots sharing following 35 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.(Lucene50CompoundReader.java:79) app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.getCompoundReader(Lucene50CompoundFormat.java:70) app//org.apache.lucene.index.SegmentCoreReaders.(SegmentCoreReaders.java:101) app//org.apache.lucene.index.SegmentReader.(SegmentReader.java:84) app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177) app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219) app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109) app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92) app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:260) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:200) app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625) app//org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1591) app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2.7% (13.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4613]' 4/10 snapshots sharing following 45 elements app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 38 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2.0% (9.9ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][AsyncLucenePersistedState#updateTask][T#1]' 7/10 snapshots sharing following 17 elements app//org.apache.lucene.index.FrozenBufferedUpdates.tryApply(FrozenBufferedUpdates.java:160) app//org.apache.lucene.index.IndexWriter.lambda$publishFrozenUpdates$3(IndexWriter.java:2592) app//org.apache.lucene.index.IndexWriter$$Lambda$4013/0x00007f5eae3ac058.process(Unknown Source) app//org.apache.lucene.index.IndexWriter.processEvents(IndexWriter.java:5116) app//org.apache.lucene.index.IndexWriter.doFlush(IndexWriter.java:3607) app//org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3566) app//org.apache.lucene.index.IndexWriter.flush(IndexWriter.java:3545) app//org.elasticsearch.gateway.PersistedClusterStateService$MetaDataIndexWriter.flush(PersistedClusterStateService.java:516) app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.updateMetaData(PersistedClusterStateService.java:689) app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.writeIncrementalStateAndCommit(PersistedClusterStateService.java:615) app//org.elasticsearch.gateway.GatewayMetaState$LucenePersistedState.setLastAcceptedState(GatewayMetaState.java:543) app//org.elasticsearch.gateway.GatewayMetaState$AsyncLucenePersistedState$1.doRun(GatewayMetaState.java:428) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 8 elements app//org.elasticsearch.gateway.PersistedClusterStateService$Writer.writeIncrementalStateAndCommit(PersistedClusterStateService.java:616) app//org.elasticsearch.gateway.GatewayMetaState$LucenePersistedState.setLastAcceptedState(GatewayMetaState.java:543) app//org.elasticsearch.gateway.GatewayMetaState$AsyncLucenePersistedState$1.doRun(GatewayMetaState.java:428) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2.0% (9.8ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4599]' 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.6% (7.8ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1013]' 3/10 snapshots sharing following 43 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.BufferedChecksumIndexInput.readByte(BufferedChecksumIndexInput.java:41) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.codecs.CodecUtil.checkHeader(CodecUtil.java:194) app//org.apache.lucene.codecs.CodecUtil.checkIndexHeader(CodecUtil.java:255) app//org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.readEntries(Lucene50CompoundReader.java:109) app//org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.(Lucene50CompoundReader.java:70) app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.getCompoundReader(Lucene50CompoundFormat.java:70) app//org.apache.lucene.index.SegmentCoreReaders.(SegmentCoreReaders.java:101) app//org.apache.lucene.index.SegmentReader.(SegmentReader.java:84) app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177) app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219) app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109) app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92) app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:260) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:200) app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625) app//org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1591) app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 33 elements app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.codecs.lucene50.Lucene50CompoundReader.(Lucene50CompoundReader.java:79) app//org.apache.lucene.codecs.lucene50.Lucene50CompoundFormat.getCompoundReader(Lucene50CompoundFormat.java:70) app//org.apache.lucene.index.SegmentCoreReaders.(SegmentCoreReaders.java:101) app//org.apache.lucene.index.SegmentReader.(SegmentReader.java:84) app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177) app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219) app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109) app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92) app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:260) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:200) app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625) app//org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1591) app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 43 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.BufferedChecksumIndexInput.readByte(BufferedChecksumIndexInput.java:41) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.codecs.CodecUtil.checkHeader(CodecUtil.java:194) app//org.apache.lucene.codecs.CodecUtil.checkIndexHeader(CodecUtil.java:255) app//org.apache.lucene.codecs.lucene70.Lucene70SegmentInfoFormat.read(Lucene70SegmentInfoFormat.java:95) app//org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:353) app//org.apache.lucene.index.SegmentInfos.readCommit(SegmentInfos.java:289) app//org.apache.lucene.index.SegmentInfos$1.doBody(SegmentInfos.java:432) app//org.apache.lucene.index.SegmentInfos$1.doBody(SegmentInfos.java:429) app//org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:680) app//org.apache.lucene.index.SegmentInfos$FindSegmentsFile.run(SegmentInfos.java:632) app//org.apache.lucene.index.SegmentInfos.readLatestCommit(SegmentInfos.java:434) app//org.elasticsearch.common.lucene.Lucene.readSegmentInfos(Lucene.java:136) app//org.elasticsearch.index.store.Store.readSegmentsInfo(Store.java:208) app//org.elasticsearch.index.store.Store.readLastCommittedSegmentsInfo(Store.java:193) app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:647) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:260) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:200) app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625) app//org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1591) app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.4% (6.9ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4589]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.2% (6.2ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4620]' 7/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.2% (6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4652]' 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.2% (6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4621]' 2/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 6/10 snapshots sharing following 2 elements java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.2% (5.9ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4641]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 2 elements java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.1% (5.5ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4655]' 10/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.1% (5.3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4664]' 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.0% (5.2ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4636]' 9/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.0% (4.9ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4639]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.0% (4.8ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4596]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 1.0% (4.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4610]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.9% (4.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4602]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.9% (4.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4590]' 6/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 44 elements app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.9% (4.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4668]' 7/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.8% (4.1ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4674]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.8% (3.8ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4603]' 2/10 snapshots sharing following 38 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 8/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.8% (3.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4635]' 8/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1045]' 3/10 snapshots sharing following 29 elements app//org.apache.lucene.codecs.compressing.CompressingStoredFieldsFormat.fieldsReader(CompressingStoredFieldsFormat.java:123) app//org.apache.lucene.codecs.lucene50.Lucene50StoredFieldsFormat.fieldsReader(Lucene50StoredFieldsFormat.java:135) app//org.apache.lucene.index.SegmentCoreReaders.(SegmentCoreReaders.java:127) app//org.apache.lucene.index.SegmentReader.(SegmentReader.java:84) app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177) app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219) app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109) app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92) app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:260) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:200) app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625) app//org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1591) app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 6/10 snapshots sharing following 26 elements app//org.apache.lucene.index.SegmentReader.(SegmentReader.java:84) app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177) app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219) app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109) app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92) app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:260) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:200) app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625) app//org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1591) app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesProducer.(Lucene80DocValuesProducer.java:90) app//org.apache.lucene.codecs.lucene80.Lucene80DocValuesFormat.fieldsProducer(Lucene80DocValuesFormat.java:146) app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat$FieldsReader.(PerFieldDocValuesFormat.java:300) app//org.apache.lucene.codecs.perfield.PerFieldDocValuesFormat.fieldsProducer(PerFieldDocValuesFormat.java:383) app//org.apache.lucene.index.SegmentDocValues.newDocValuesProducer(SegmentDocValues.java:52) app//org.apache.lucene.index.SegmentDocValues.getDocValuesProducer(SegmentDocValues.java:68) app//org.apache.lucene.index.SegmentReader.initDocValuesProducer(SegmentReader.java:176) app//org.apache.lucene.index.SegmentReader.(SegmentReader.java:100) app//org.apache.lucene.index.ReadersAndUpdates.getReader(ReadersAndUpdates.java:177) app//org.apache.lucene.index.ReadersAndUpdates.getReadOnlyClone(ReadersAndUpdates.java:219) app//org.apache.lucene.index.StandardDirectoryReader.open(StandardDirectoryReader.java:109) app//org.apache.lucene.index.IndexWriter.getReader(IndexWriter.java:526) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:116) app//org.apache.lucene.index.DirectoryReader.open(DirectoryReader.java:92) app//org.elasticsearch.index.engine.InternalEngine.createReaderManager(InternalEngine.java:644) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:260) app//org.elasticsearch.index.engine.InternalEngine.(InternalEngine.java:200) app//org.elasticsearch.index.engine.InternalEngineFactory.newReadWriteEngine(InternalEngineFactory.java:25) app//org.elasticsearch.index.shard.IndexShard.innerOpenEngineAndTranslog(IndexShard.java:1625) app//org.elasticsearch.index.shard.IndexShard.openEngineAndRecoverFromTranslog(IndexShard.java:1591) app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4618]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4642]' 7/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.5ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4595]' 7/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4673]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4629]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.2ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4584]' 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.7% (3.2ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4593]' 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.6% (3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4516]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.6% (3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4634]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.6% (2.9ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4633]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.6% (2.9ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4651]' 4/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.6% (2.8ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4598]' 3/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.6% (2.8ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4647]' 7/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.5% (2.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4631]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.5% (2.6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4660]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.5% (2.5ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4630]' 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.5% (2.3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4657]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.5% (2.3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4597]' 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.5% (2.2ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4626]' 2/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 38 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.4% (2ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4627]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.4% (2ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4644]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.4% (1.8ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4649]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.4% (1.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4591]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.7ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4608]' 4/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:160) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:849) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4600]' 7/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.6ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4622]' 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 6/10 snapshots sharing following 2 elements java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.5ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4594]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.5ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4604]' 3/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 7/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4609]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 5/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4638]' 3/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 7/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.4ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4614]' 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4653]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4643]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.3% (1.3ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4580]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 5/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.2% (1.1ms out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4601]' 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 2 elements java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.2% (992micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4586]' 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.2% (981.7micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4606]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.2% (955.1micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1095]' 4/10 snapshots sharing following 63 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.phase1(RecoverySourceHandler.java:452) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$recoverToTarget$11(RecoverySourceHandler.java:273) app//org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$4837/0x00007f5b9c80c168.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:112) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) app//org.elasticsearch.action.StepListener.whenComplete(StepListener.java:78) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.recoverToTarget(RecoverySourceHandler.java:271) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.recover(PeerRecoverySourceService.java:121) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.access$000(PeerRecoverySourceService.java:56) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:127) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:124) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 61 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.phase1(RecoverySourceHandler.java:452) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$recoverToTarget$11(RecoverySourceHandler.java:273) app//org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$4837/0x00007f5b9c80c168.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:112) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) app//org.elasticsearch.action.StepListener.whenComplete(StepListener.java:78) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.recoverToTarget(RecoverySourceHandler.java:271) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.recover(PeerRecoverySourceService.java:121) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.access$000(PeerRecoverySourceService.java:56) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:127) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:124) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 54 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.phase1(RecoverySourceHandler.java:452) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$recoverToTarget$11(RecoverySourceHandler.java:273) app//org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$4837/0x00007f5b9c80c168.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:112) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) app//org.elasticsearch.action.StepListener.whenComplete(StepListener.java:78) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.recoverToTarget(RecoverySourceHandler.java:271) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.recover(PeerRecoverySourceService.java:121) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.access$000(PeerRecoverySourceService.java:56) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:127) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:124) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.fs.UnixNativeDispatcher.open0(Native Method) java.base@14/sun.nio.fs.UnixNativeDispatcher.open(UnixNativeDispatcher.java:68) java.base@14/sun.nio.fs.UnixChannelFactory.open(UnixChannelFactory.java:267) java.base@14/sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:143) java.base@14/sun.nio.fs.UnixChannelFactory.newFileChannel(UnixChannelFactory.java:156) java.base@14/sun.nio.fs.UnixFileSystemProvider.newFileChannel(UnixFileSystemProvider.java:180) java.base@14/java.nio.channels.FileChannel.open(FileChannel.java:292) java.base@14/java.nio.channels.FileChannel.open(FileChannel.java:345) app//org.apache.lucene.store.NIOFSDirectory.openInput(NIOFSDirectory.java:81) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:143) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.phase1(RecoverySourceHandler.java:452) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.lambda$recoverToTarget$11(RecoverySourceHandler.java:273) app//org.elasticsearch.indices.recovery.RecoverySourceHandler$$Lambda$4837/0x00007f5b9c80c168.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) app//org.elasticsearch.common.util.concurrent.ListenableFuture$1.doRun(ListenableFuture.java:112) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) app//org.elasticsearch.common.util.concurrent.ListenableFuture.notifyListener(ListenableFuture.java:106) app//org.elasticsearch.common.util.concurrent.ListenableFuture.addListener(ListenableFuture.java:68) app//org.elasticsearch.action.StepListener.whenComplete(StepListener.java:78) app//org.elasticsearch.indices.recovery.RecoverySourceHandler.recoverToTarget(RecoverySourceHandler.java:271) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.recover(PeerRecoverySourceService.java:121) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService.access$000(PeerRecoverySourceService.java:56) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:127) app//org.elasticsearch.indices.recovery.PeerRecoverySourceService$StartRecoveryTransportRequestHandler.messageReceived(PeerRecoverySourceService.java:124) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.2% (891micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4616]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.2% (885.7micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4623]' 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.2% (875.4micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4637]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (749.2micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4640]' 7/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (729.4micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4605]' 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (668.7micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1023]' 2/10 snapshots sharing following 13 elements app//org.elasticsearch.index.shard.StoreRecovery.internalRecoverFromStore(StoreRecovery.java:432) app//org.elasticsearch.index.shard.StoreRecovery.lambda$recoverFromStore$0(StoreRecovery.java:98) app//org.elasticsearch.index.shard.StoreRecovery$$Lambda$4472/0x00007f5b9ecf84b0.get(Unknown Source) app//org.elasticsearch.action.ActionListener.completeWith(ActionListener.java:325) app//org.elasticsearch.index.shard.StoreRecovery.recoverFromStore(StoreRecovery.java:96) app//org.elasticsearch.index.shard.IndexShard.recoverFromStore(IndexShard.java:1865) app//org.elasticsearch.index.shard.IndexShard$$Lambda$4467/0x00007f5b9ecfb0b0.accept(Unknown Source) app//org.elasticsearch.action.ActionRunnable$2.doRun(ActionRunnable.java:73) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 8/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (594.8micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4624]' 3/10 snapshots sharing following 44 elements app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 38 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (544.9micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4646]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (518.9micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4583]' 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (299.8micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4628]' 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 4/10 snapshots sharing following 2 elements java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (278.1micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4585]' 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:160) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:849) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.1% (266.5micros out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4582]' 3/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 5/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (25.9micros out of 500ms) cpu usage by tstateead 'ticker-schedule-trigger-engine' 10/10 snapshots sharing following 2 elements java.base@14/java.lang.Tstateead.sleep(Native Method) org.elasticsearch.xpack.watcher.trigger.schedule.engine.TickerScheduleTriggerEngine$Ticker.run(TickerScheduleTriggerEngine.java:193) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4581]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4587]' 4/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 6/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4671]' 2/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 39 elements app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 6/10 snapshots sharing following 2 elements java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][clusterApplierService#updateTask][T#1]' 2/10 snapshots sharing following 18 elements app//org.elasticsearch.cluster.routing.RoutingNodes.(RoutingNodes.java:87) app//org.elasticsearch.cluster.ClusterState.getRoutingNodes(ClusterState.java:320) app//org.elasticsearch.indices.cluster.IndicesClusterStateService.updateFailedShardsCache(IndicesClusterStateService.java:259) app//org.elasticsearch.indices.cluster.IndicesClusterStateService.applyClusterState(IndicesClusterStateService.java:234) app//org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateAppliers$5(ClusterApplierService.java:517) app//org.elasticsearch.cluster.service.ClusterApplierService$$Lambda$4256/0x00007f5d795180b0.accept(Unknown Source) java.base@14/java.lang.Iterable.forEach(Iterable.java:75) app//org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateAppliers(ClusterApplierService.java:514) app//org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:485) app//org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:432) app//org.elasticsearch.cluster.service.ClusterApplierService.access$100(ClusterApplierService.java:73) app//org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:176) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633) app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsTstateeadPoolExecutor.java:252) app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsTstateeadPoolExecutor.java:215) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 16 elements app//org.elasticsearch.cluster.service.ClusterApplierService.lambda$callClusterStateListeners$6(ClusterApplierService.java:527) app//org.elasticsearch.cluster.service.ClusterApplierService$$Lambda$4281/0x00007f5d7970b4b0.accept(Unknown Source) java.base@14/java.util.Spliterators$ArraySpliterator.forEachRemaining(Spliterators.java:948) java.base@14/java.util.stream.Streams$ConcatSpliterator.forEachRemaining(Streams.java:734) java.base@14/java.util.stream.ReferencePipeline$Head.forEach(ReferencePipeline.java:658) app//org.elasticsearch.cluster.service.ClusterApplierService.callClusterStateListeners(ClusterApplierService.java:523) app//org.elasticsearch.cluster.service.ClusterApplierService.applyChanges(ClusterApplierService.java:498) app//org.elasticsearch.cluster.service.ClusterApplierService.runTask(ClusterApplierService.java:432) app//org.elasticsearch.cluster.service.ClusterApplierService.access$100(ClusterApplierService.java:73) app//org.elasticsearch.cluster.service.ClusterApplierService$UpdateTask.run(ClusterApplierService.java:176) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingRunnable.run(TstateeadContext.java:633) app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.runAndClean(PrioritizedEsTstateeadPoolExecutor.java:252) app//org.elasticsearch.common.util.concurrent.PrioritizedEsTstateeadPoolExecutor$TieBreakingPrioritizedRunnable.run(PrioritizedEsTstateeadPoolExecutor.java:215) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 10 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.park(LockSupport.java:341) java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer$ConditionNode.block(AbstractQueuedSyncstateonizer.java:505) java.base@14/java.util.concurrent.ForkJoinPool.managedBlock(ForkJoinPool.java:3137) java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer$ConditionObject.await(AbstractQueuedSyncstateonizer.java:1614) java.base@14/java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:548) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1056) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'ml-cpp-log-tail-tstateead' 10/10 snapshots sharing following 6 elements java.base@14/java.io.FileInputStream.readBytes(Native Method) java.base@14/java.io.FileInputStream.read(FileInputStream.java:250) org.elasticsearch.xpack.ml.process.logging.CppLogMessageHandler.tailStream(CppLogMessageHandler.java:105) org.elasticsearch.xpack.ml.process.NativeController.lambda$tailLogsInTstateead$0(NativeController.java:72) org.elasticsearch.xpack.ml.process.NativeController$$Lambda$2548/0x00007f5ee420a440.run(Unknown Source) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'DestroyJavaVM' unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1041]' 2/10 snapshots sharing following 35 elements app//org.elasticsearch.cluster.metadata.MetaData$MetaDataDiff.apply(MetaData.java:962) app//org.elasticsearch.cluster.metadata.MetaData$MetaDataDiff.apply(MetaData.java:878) app//org.elasticsearch.cluster.ClusterState$ClusterStateDiff.apply(ClusterState.java:919) app//org.elasticsearch.cluster.ClusterState$ClusterStateDiff.apply(ClusterState.java:841) app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.handleIncomingPublishRequest(PublicationTransportHandler.java:421) app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$new$0(PublicationTransportHandler.java:100) app//org.elasticsearch.cluster.coordination.PublicationTransportHandler$$Lambda$3123/0x00007f5eaf5d00b0.messageReceived(Unknown Source) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 7/10 snapshots sharing following 9 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'Connection evictor' 10/10 snapshots sharing following 3 elements java.base@14/java.lang.Tstateead.sleep(Native Method) org.apache.http.impl.client.IdleConnectionEvictor$1.run(IdleConnectionEvictor.java:66) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1092]' 10/10 snapshots sharing following 51 elements app//org.elasticsearch.monitor.jvm.HotTstateeads.detect(HotTstateeads.java:90) app//org.elasticsearch.action.admin.cluster.node.hottstateeads.TransportNodesHotTstateeadsAction.nodeOperation(TransportNodesHotTstateeadsAction.java:75) app//org.elasticsearch.action.admin.cluster.node.hottstateeads.TransportNodesHotTstateeadsAction.nodeOperation(TransportNodesHotTstateeadsAction.java:38) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorizeAction$4(AuthorizationService.java:247) org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$4245/0x00007f5d795130b0.accept(Unknown Source) org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:625) org.elasticsearch.xpack.security.authz.AuthorizationService$AuthorizationResultListener.onResponse(AuthorizationService.java:600) app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) org.elasticsearch.xpack.security.authz.RBACEngine.authorizeClusterAction(RBACEngine.java:147) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeAction(AuthorizationService.java:249) org.elasticsearch.xpack.security.authz.AuthorizationService.maybeAuthorizeRunAs(AuthorizationService.java:232) org.elasticsearch.xpack.security.authz.AuthorizationService.lambda$authorize$1(AuthorizationService.java:197) org.elasticsearch.xpack.security.authz.AuthorizationService$$Lambda$4216/0x00007f5e1a4b1560.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) app//org.elasticsearch.action.support.ContextPreservingActionListener.onResponse(ContextPreservingActionListener.java:43) org.elasticsearch.xpack.security.authz.RBACEngine.lambda$resolveAuthorizationInfo$1(RBACEngine.java:120) org.elasticsearch.xpack.security.authz.RBACEngine$$Lambda$4218/0x00007f5e1a4b2d08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.store.CompositeRolesStore.getRoles(CompositeRolesStore.java:249) org.elasticsearch.xpack.security.authz.RBACEngine.getRoles(RBACEngine.java:126) org.elasticsearch.xpack.security.authz.RBACEngine.resolveAuthorizationInfo(RBACEngine.java:114) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:199) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4648]' 6/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'process reaper' 10/10 snapshots sharing following 5 elements java.base@14/java.lang.ProcessHandleImpl.waitForProcessExit0(Native Method) java.base@14/java.lang.ProcessHandleImpl$1.run(ProcessHandleImpl.java:138) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[keepAlive/7.7.0]' 10/10 snapshots sharing following 7 elements java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.park(LockSupport.java:211) java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer.acquire(AbstractQueuedSyncstateonizer.java:714) java.base@14/java.util.concurrent.locks.AbstractQueuedSyncstateonizer.acquireSharedInterruptibly(AbstractQueuedSyncstateonizer.java:1046) java.base@14/java.util.concurrent.CountDownLatch.await(CountDownLatch.java:232) app//org.elasticsearch.bootstrap.Bootstrap$1.run(Bootstrap.java:89) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'Common-Cleaner' 10/10 snapshots sharing following 5 elements java.base@14/java.lang.Object.wait(Native Method) java.base@14/java.lang.ref.ReferenceQueue.remove(ReferenceQueue.java:155) java.base@14/jdk.internal.ref.CleanerImpl.run(CleanerImpl.java:148) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) java.base@14/jdk.internal.misc.InnocuousTstateead.run(InnocuousTstateead.java:134) 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][fetch_shard_store][T#4661]' 2/10 snapshots sharing following 53 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readByte(BufferedIndexInput.java:54) app//org.apache.lucene.store.DataInput.readInt(DataInput.java:102) app//org.apache.lucene.store.BufferedIndexInput.readInt(BufferedIndexInput.java:183) app//org.apache.lucene.codecs.CodecUtil.validateFooter(CodecUtil.java:500) app//org.apache.lucene.codecs.CodecUtil.retrieveChecksum(CodecUtil.java:487) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:893) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 55 elements java.base@14/sun.nio.ch.FileDispatcherImpl.pread0(Native Method) java.base@14/sun.nio.ch.FileDispatcherImpl.pread(FileDispatcherImpl.java:54) java.base@14/sun.nio.ch.IOUtil.readIntoNativeBuffer(IOUtil.java:274) java.base@14/sun.nio.ch.IOUtil.read(IOUtil.java:245) java.base@14/sun.nio.ch.FileChannelImpl.readInternal(FileChannelImpl.java:817) java.base@14/sun.nio.ch.FileChannelImpl.read(FileChannelImpl.java:802) app//org.apache.lucene.store.NIOFSDirectory$NIOFSIndexInput.readInternal(NIOFSDirectory.java:179) app//org.apache.lucene.store.BufferedIndexInput.refill(BufferedIndexInput.java:342) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:140) app//org.apache.lucene.store.BufferedIndexInput.readBytes(BufferedIndexInput.java:116) app//org.elasticsearch.index.store.Store$VerifyingIndexInput.readBytes(Store.java:1253) app//org.apache.lucene.store.DataInput.readBytes(DataInput.java:88) app//org.elasticsearch.common.lucene.store.InputStreamIndexInput.read(InputStreamIndexInput.java:73) app//org.elasticsearch.common.io.Streams.readFully(Streams.java:199) app//org.elasticsearch.index.store.Store$MetadataSnapshot.hashFile(Store.java:911) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:890) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 3/10 snapshots sharing following 46 elements java.base@14/java.nio.MappedByteBuffer.load(MappedByteBuffer.java:252) app//org.apache.lucene.store.MMapDirectory.map(MMapDirectory.java:272) app//org.apache.lucene.store.MMapDirectory.openInput(MMapDirectory.java:242) app//org.elasticsearch.index.store.FsDirectoryFactory$HybridDirectory.openInput(FsDirectoryFactory.java:141) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.apache.lucene.store.FilterDirectory.openInput(FilterDirectory.java:100) app//org.elasticsearch.index.store.Store$MetadataSnapshot.checksumFromLuceneFile(Store.java:878) app//org.elasticsearch.index.store.Store$MetadataSnapshot.loadMetadata(Store.java:841) app//org.elasticsearch.index.store.Store$MetadataSnapshot.(Store.java:759) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:282) app//org.elasticsearch.index.store.Store.getMetadata(Store.java:247) app//org.elasticsearch.index.shard.IndexShard.snapshotStoreMetadata(IndexShard.java:1214) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.listStoreMetaData(TransportNodesListShardStoreMetaData.java:135) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:117) app//org.elasticsearch.indices.store.TransportNodesListShardStoreMetaData.nodeOperation(TransportNodesListShardStoreMetaData.java:67) app//org.elasticsearch.action.support.nodes.TransportNodesAction.nodeOperation(TransportNodesAction.java:129) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:244) app//org.elasticsearch.action.support.nodes.TransportNodesAction$NodeTransportHandler.messageReceived(TransportNodesAction.java:240) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 2/10 snapshots sharing following 2 elements java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) 0.0% (0s out of 500ms) cpu usage by tstateead 'Notification Tstateead' unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot unique snapshot 0.0% (0s out of 500ms) cpu usage by tstateead 'elasticsearch[serverra2_warm.sit.comp.state][generic][T#1063]' 2/10 snapshots sharing following 35 elements app//org.elasticsearch.cluster.metadata.MetaData$MetaDataDiff.apply(MetaData.java:962) app//org.elasticsearch.cluster.metadata.MetaData$MetaDataDiff.apply(MetaData.java:878) app//org.elasticsearch.cluster.ClusterState$ClusterStateDiff.apply(ClusterState.java:919) app//org.elasticsearch.cluster.ClusterState$ClusterStateDiff.apply(ClusterState.java:841) app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.handleIncomingPublishRequest(PublicationTransportHandler.java:421) app//org.elasticsearch.cluster.coordination.PublicationTransportHandler.lambda$new$0(PublicationTransportHandler.java:100) app//org.elasticsearch.cluster.coordination.PublicationTransportHandler$$Lambda$3123/0x00007f5eaf5d00b0.messageReceived(Unknown Source) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$1.doRun(SecurityServerTransportInterceptor.java:257) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) app//org.elasticsearch.common.util.concurrent.EsExecutors$DirectExecutorService.execute(EsExecutors.java:225) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.lambda$messageReceived$0(SecurityServerTransportInterceptor.java:306) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler$$Lambda$4170/0x00007f5e1a2edd08.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authz.AuthorizationService.authorizeSystemUser(AuthorizationService.java:385) org.elasticsearch.xpack.security.authz.AuthorizationService.authorize(AuthorizationService.java:190) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.lambda$inbound$1(ServerTransportFilter.java:129) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile$$Lambda$4172/0x00007f5e1a2ec568.accept(Unknown Source) app//org.elasticsearch.action.ActionListener$1.onResponse(ActionListener.java:63) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$authenticateAsync$2(AuthenticationService.java:324) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4174/0x00007f5e1a2eacb0.accept(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lambda$lookForExistingAuthentication$6(AuthenticationService.java:386) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator$$Lambda$4175/0x00007f5e1a2e90b0.run(Unknown Source) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.lookForExistingAuthentication(AuthenticationService.java:397) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.authenticateAsync(AuthenticationService.java:321) org.elasticsearch.xpack.security.authc.AuthenticationService$Authenticator.access$000(AuthenticationService.java:263) org.elasticsearch.xpack.security.authc.AuthenticationService.authenticate(AuthenticationService.java:174) org.elasticsearch.xpack.security.transport.ServerTransportFilter$NodeProfile.inbound(ServerTransportFilter.java:120) org.elasticsearch.xpack.security.transport.SecurityServerTransportInterceptor$ProfileSecuredRequestHandler.messageReceived(SecurityServerTransportInterceptor.java:313) app//org.elasticsearch.transport.RequestHandlerRegistry.processMessageReceived(RequestHandlerRegistry.java:63) app//org.elasticsearch.transport.InboundHandler$RequestHandler.doRun(InboundHandler.java:264) app//org.elasticsearch.common.util.concurrent.TstateeadContext$ContextPreservingAbstractRunnable.doRun(TstateeadContext.java:692) app//org.elasticsearch.common.util.concurrent.AbstractRunnable.run(AbstractRunnable.java:37) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1130) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832) unique snapshot java.base@14/jdk.internal.misc.Unsafe.park(Native Method) java.base@14/java.util.concurrent.locks.LockSupport.parkNanos(LockSupport.java:252) java.base@14/java.util.concurrent.LinkedTransferQueue.awaitMatch(LinkedTransferQueue.java:740) java.base@14/java.util.concurrent.LinkedTransferQueue.xfer(LinkedTransferQueue.java:684) java.base@14/java.util.concurrent.LinkedTransferQueue.poll(LinkedTransferQueue.java:1374) java.base@14/java.util.concurrent.TstateeadPoolExecutor.getTask(TstateeadPoolExecutor.java:1055) java.base@14/java.util.concurrent.TstateeadPoolExecutor.runWorker(TstateeadPoolExecutor.java:1116) java.base@14/java.util.concurrent.TstateeadPoolExecutor$Worker.run(TstateeadPoolExecutor.java:630) java.base@14/java.lang.Tstateead.run(Tstateead.java:832)