【Druid】进程占用CPU过高，问题排查过程

原创
巧克力黒 2023-04-20 16:58:46 博主文章分类：Druid ©著作权
文章标签 druid cpu 文章分类 运维
©著作权归作者所有：来自51CTO博客作者巧克力黒的原创作品，请联系作者获取转载授权，否则将追究法律责任
[data_node@data-druid-ad_data bin]$ top -Hp 10839
top - 20:16:19 up 926 days,  7:34,  1 user,  load average: 38.77, 33.87, 24.96
Threads: 330 total,   3 running, 327 sleeping,   0 stopped,   0 zombie
%Cpu(s):  4.3 us,  1.5 sy,  0.0 ni, 94.2 id,  0.0 wa,  0.0 hi,  0.0 si,  0.0 st
KiB Mem : 39459468+total, 13903212 free, 16936230+used, 21132918+buff/cache
KiB Swap:        0 total,        0 free,        0 used. 22321147+avail Mem

PID USER      PR  NI    VIRT    RES    SHR S %CPU %MEM     TIME+ COMMAND
16878 data_node     20   0 37.808g 0.014t 1.243g R 95.0  3.9   6:48.40 appenderator_me
11861 data_node     20   0 37.808g 0.014t 1.243g R 94.7  3.9   9:52.52 ad_flow_data-incre
11342 data_node     20   0 37.808g 0.014t 1.243g R 79.1  3.9   8:08.43 task-runner-0-p
10848 data_node     20   0 37.808g 0.014t 1.243g S  2.7  3.9  10:44.02 java
10851 data_node     20   0 37.808g 0.014t 1.243g S  2.7  3.9  10:43.07 java
10852 data_node     20   0 37.808g 0.014t 1.243g S  2.7  3.9  10:42.00 java
10854 data_node     20   0 37.808g 0.014t 1.243g S  2.7  3.9  10:42.66 java

[data_node@data-druid-ad_data bin]$ printf "%x\n"  16878
41ee


[data_node@data-druid-ad_data bin]$ ./jstack -l 10839 > 10839.txt
[data_node@data-druid-ad_data bin]$ vim 10839.txt


[data_node@data-druid-ad_data bin]$ cat 10839.txt | grep -A 100 41ee
"appenderator_merge_0" #343 daemon prio=5 os_prio=0 tid=0x00007f37244ae000 nid=0x41ee runnable [0x00007f3d693c9000]
   java.lang.Thread.State: RUNNABLE
	at sun.nio.ch.FileDispatcherImpl.write0(Native Method)
	at sun.nio.ch.FileDispatcherImpl.write(FileDispatcherImpl.java:60)
	at sun.nio.ch.IOUtil.writeFromNativeBuffer(IOUtil.java:93)
	at sun.nio.ch.IOUtil.write(IOUtil.java:65)
	at sun.nio.ch.FileChannelImpl.write(FileChannelImpl.java:211)
	- locked <0x0000000755a47788> (a java.lang.Object)
	at org.apache.druid.io.Channels.writeFully(Channels.java:32)
	at org.apache.druid.segment.writeout.FileWriteOutBytes.flush(FileWriteOutBytes.java:60)
	at org.apache.druid.segment.writeout.FileWriteOutBytes.size(FileWriteOutBytes.java:108)
	at org.apache.druid.segment.data.GenericIndexedWriter.getSerializedSize(GenericIndexedWriter.java:279)
	at org.apache.druid.segment.data.GenericIndexedWriter.write(GenericIndexedWriter.java:231)
	at org.apache.druid.segment.serde.LargeColumnSupportedComplexColumnSerializer.serialize(LargeColumnSupportedComplexColumnSerializer.java:105)
	at org.apache.druid.segment.IndexMergerV9.mergeIndexesAndWriteColumns(IndexMergerV9.java:499)
	at org.apache.druid.segment.IndexMergerV9.makeIndexFiles(IndexMergerV9.java:192)
	at org.apache.druid.segment.IndexMergerV9.merge(IndexMergerV9.java:915)
	at org.apache.druid.segment.IndexMergerV9.mergeQueryableIndex(IndexMergerV9.java:833)
	at org.apache.druid.segment.IndexMergerV9.mergeQueryableIndex(IndexMergerV9.java:811)
	at org.apache.druid.segment.realtime.appenderator.AppenderatorImpl.mergeAndPush(AppenderatorImpl.java:729)
	at org.apache.druid.segment.realtime.appenderator.AppenderatorImpl.lambda$1(AppenderatorImpl.java:633)
	at org.apache.druid.segment.realtime.appenderator.AppenderatorImpl$$Lambda$466/2061454348.apply(Unknown Source)
	at com.google.common.util.concurrent.Futures$1.apply(Futures.java:713)
	at com.google.common.util.concurrent.Futures$ChainingListenableFuture.run(Futures.java:861)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

   Locked ownable synchronizers:
	- <0x0000000653b705c8> (a java.util.concurrent.ThreadPoolExecutor$Worker)

"appenderator_persist_0" #308 daemon prio=5 os_prio=0 tid=0x00007f385426f800 nid=0x2e55 waiting on condition [0x00007f3d6a2eb000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000006523ed1f8> (a java.util.concurrent.SynchronousQueue$TransferStack)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.SynchronousQueue$TransferStack.awaitFulfill(SynchronousQueue.java:458)
	at java.util.concurrent.SynchronousQueue$TransferStack.transfer(SynchronousQueue.java:362)
	at java.util.concurrent.SynchronousQueue.take(SynchronousQueue.java:924)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

   Locked ownable synchronizers:
	- None

"processing-2" #306 daemon prio=5 os_prio=0 tid=0x00007f37cc05a800 nid=0x2cb1 waiting on condition [0x00007f3d69de8000]
   java.lang.Thread.State: WAITING (parking)
	at sun.misc.Unsafe.park(Native Method)
	- parking to wait for  <0x00000005ca9f5c48> (a java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject)
	at java.util.concurrent.locks.LockSupport.park(LockSupport.java:175)
	at java.util.concurrent.locks.AbstractQueuedSynchronizer$ConditionObject.await(AbstractQueuedSynchronizer.java:2039)
	at java.util.concurrent.PriorityBlockingQueue.take(PriorityBlockingQueue.java:549)
	at java.util.concurrent.ThreadPoolExecutor.getTask(ThreadPoolExecutor.java:1074)
	at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1134)
	at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
	at java.lang.Thread.run(Thread.java:748)

   Locked ownable synchronizers:
	- None

"groupBy_bi_ad_data_ad_[2023-03-15T19:00:00.000Z/2023-03-15T20:00:00.000Z]" #305 daemon prio=5 os_prio=0 tid=0x00007f37f0033800 nid=0x2cb0 runnable [0x00007f3d69ee8000]
   java.lang.Thread.State: RUNNABLE
	at org.apache.druid.query.aggregation.Sad_dataleDoubleBufferAggregator.aggregate(Sad_dataleDoubleBufferAggregator.java:56)
	at org.apache.druid.query.aggregation.AggregatorAdapters.aggregateBuffered(AggregatorAdapters.java:164)
	at org.apache.druid.query.groupby.epinephelinae.AbstractBufferHashGrouper.aggregate(AbstractBufferHashGrouper.java:162)
	at org.apache.druid.query.groupby.epinephelinae.Grouper.aggregate(Grouper.java:85)
	at org.apache.druid.query.groupby.epinephelinae.GroupByQueryEngineV2$HashAggregateIterator.aggregateSingleValueDims(GroupByQueryEngineV2.java:554)
	at org.apache.druid.query.groupby.epinephelinae.GroupByQueryEngineV2$GroupByEngineIterator.initNewDelegate(GroupByQueryEngineV2.java:397)
	at org.apache.druid.query.groupby.epinephelinae.GroupByQueryEngineV2$GroupByEngineIterator.hasNext(GroupByQueryEngineV2.java:443)
	at org.apache.druid.java.util.common.guava.BaseSequence.accumulate(BaseSequence.java:44)
	at org.apache.druid.java.util.common.guava.ConcatSequence.lambda$accumulate$0(ConcatSequence.java:41)
	at org.apache.druid.java.util.common.guava.ConcatSequence$$Lambda$270/1775145376.accumulate(Unknown Source)
	at org.apache.druid.java.util.common.guava.MappingAccumulator.accumulate(MappingAccumulator.java:40)
	at org.apache.druid.java.util.common.guava.FilteringAccumulator.accumulate(FilteringAccumulator.java:41)
	at org.apache.druid.java.util.common.guava.MappingAccumulator.accumulate(MappingAccumulator.java:40)
	at org.apache.druid.java.util.common.guava.BaseSequence.accumulate(BaseSequence.java:45)
	at org.apache.druid.java.util.common.guava.MappedSequence.accumulate(MappedSequence.java:43)
	at org.apache.druid.java.util.common.guava.WrappingSequence$1.get(WrappingSequence.java:50)
	at org.apache.druid.java.util.common.guava.SequenceWrapper.wrap(SequenceWrapper.java:55)
	at org.apache.druid.java.util.common.guava.WrappingSequence.accumulate(WrappingSequence.java:45)
	at org.apache.druid.java.util.common.guava.FilteredSequence.accumulate(FilteredSequence.java:45)
	at org.apache.druid.java.util.common.guava.MappedSequence.accumulate(MappedSequence.java:43)
	at org.apache.druid.java.util.common.guava.ConcatSequence.accumulate(ConcatSequence.java:41)
	at org.apache.druid.java.util.common.guava.WrappingSequence$1.get(WrappingSequence.java:50)
	at org.apache.druid.java.util.common.guava.SequenceWrapper.wrap(SequenceWrapper.java:55)
	at org.apache.druid.java.util.common.guava.WrappingSequence.accumulate(WrappingSequence.java:45)
	at org.apache.druid.java.util.common.guava.WrappingSequence$1.get(WrappingSequence.java:50)
	at org.apache.druid.java.util.common.guava.SequenceWrapper.wrap(SequenceWrapper.java:55)
	at org.apache.druid.java.util.common.guava.WrappingSequence.accumulate(WrappingSequence.java:45)
	at org.apache.druid.java.util.common.guava.Sequence.toList(Sequence.java:85)
	at org.apache.druid.query.ChainedExecutionQueryRunner$1$1.call(ChainedExecutionQueryRunner.java:124)
	at org.apache.druid.query.ChainedExecutionQueryRunner$1$1.call(ChainedExecutionQueryRunner.java:114)
	at java.util.concurrent.FutureTask.run(FutureTask.java:266)
	at org.apache.druid.java.util.common.concurrent.DirectExecutorService.execute(DirectExecutorService.java:81)
	at java.util.concurrent.AbstractExecutorService.submit(AbstractExecutorService.java:134)
	at com.google.common.util.concurrent.AbstractListeningExecutorService.submit(AbstractListeningExecutorService.java:58)
	at org.apache.druid.query.ChainedExecutionQueryRunner$1.lambda$make$0(ChainedExecutionQueryRunner.java:112)
	at org.apache.druid.query.ChainedExecutionQueryRunner$1$$Lambda$245/118522997.apply(Unknown Source)
	at com.google.common.collect.Iterators$8.transform(Iterators.java:794)


[data_node@data-druid-ad_data bin]$ printf "%x\n"  11861
2e55