本文基于dubbo 2.7.5版本代码

在一些场景中,比如服务端收到请求需要在业务线程池中处理请求时,dubbo需要通过ExecutorRepository创建线程池。在创建线程池的时候,dubbo设置了RejectedExecutionHandler,也就是当线程池满的时候,会调用RejectedExecutionHandler。dubbo提供了RejectedExecutionHandler的实现类AbortPolicyWithReport。当线程池满的时候,AbortPolicyWithReport将线程的堆栈信息打印到执行的文件中。下面来介绍一下AbortPolicyWithReport的实现。

在dubbo中可以通过dump.directory指定打印堆栈信息的目录,如果不设置,dubbo将堆栈信息打印到System.getProperty(“user.home”)目录下。设置dump.directory的方式可以有:

  • @Service(parameters = {“dump.directory”,"/tmp"}),这个设置只对单个的服务生效
  • dubbo.application.parameters[dump.directory]=/tmp,这样设置会使整个应用的所有服务都生效

下面是AbortPolicyWithReport源码。

public class AbortPolicyWithReport extends ThreadPoolExecutor.AbortPolicy {
    protected static final Logger logger = LoggerFactory.getLogger(AbortPolicyWithReport.class);
	//threadName可以作为日志打印的关键字,默认是Dubbo,可以通过参数“threadname”设置
    private final String threadName;
	//url是由服务参数组成的,
	//如果是服务端创建AbortPolicyWithReport ,则url是服务端发布服务的url
	//如果是消费端,则url代表了消费端需要引用的远程服务url
    private final URL url;
	//表示最后一次打印堆栈信息的事件,dubbo每10分钟打印一次
    private static volatile long lastPrintTime = 0;
	//表示堆栈信息的打印间隔
    private static final long TEN_MINUTES_MILLS = 10 * 60 * 1000;
    private static final String OS_WIN_PREFIX = "win";
    private static final String OS_NAME_KEY = "os.name";
    private static final String WIN_DATETIME_FORMAT = "yyyy-MM-dd_HH-mm-ss";
    private static final String DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd_HH:mm:ss";
    private static Semaphore guard = new Semaphore(1);

    public AbortPolicyWithReport(String threadName, URL url) {
        this.threadName = threadName;
        this.url = url;
    }

    @Override
    public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
        String msg = String.format("Thread pool is EXHAUSTED!" +
                " Thread Name: %s, Pool Size: %d (active: %d, core: %d, max: %d, largest: %d), Task: %d (completed: "
                + "%d)," +
                " Executor status:(isShutdown:%s, isTerminated:%s, isTerminating:%s), in %s://%s:%d!",
            threadName, e.getPoolSize(), e.getActiveCount(), e.getCorePoolSize(), e.getMaximumPoolSize(),
            e.getLargestPoolSize(),
            e.getTaskCount(), e.getCompletedTaskCount(), e.isShutdown(), e.isTerminated(), e.isTerminating(),
            url.getProtocol(), url.getIp(), url.getPort());
        //打印日志,其中属性theadName会打印到日志中
        logger.warn(msg);
        //调用打印堆栈信息的方法
        dumpJStack();
        throw new RejectedExecutionException(msg);
    }

    private void dumpJStack() {
        long now = System.currentTimeMillis();

        //一个线程池每10分钟打印一次堆栈信息
        if (now - lastPrintTime < TEN_MINUTES_MILLS) {
            return;
        }
		//guard是信号量,同时只能一个线程打印堆栈
        if (!guard.tryAcquire()) {
            return;
        }
		
        ExecutorService pool = Executors.newSingleThreadExecutor();
        //使用java原生线程池打印堆栈信息,这个线程池中只有一个线程,用完就关闭
        pool.execute(() -> {
        	//获得的打印堆栈信息文件的目录
            String dumpPath = url.getParameter(DUMP_DIRECTORY, System.getProperty("user.home"));

            SimpleDateFormat sdf;

            String os = System.getProperty(OS_NAME_KEY).toLowerCase();

            // 不同系统的时间格式不同
            if (os.contains(OS_WIN_PREFIX)) {
                sdf = new SimpleDateFormat(WIN_DATETIME_FORMAT);
            } else {
                sdf = new SimpleDateFormat(DEFAULT_DATETIME_FORMAT);
            }

            String dateStr = sdf.format(new Date());
            //try-with-resources
            //打印的文件名为:Dubbo_JStack.log.时间,其中时间到秒
            try (FileOutputStream jStackStream = new FileOutputStream(
                new File(dumpPath, "Dubbo_JStack.log" + "." + dateStr))) {
                //使用JVMUtil打印,代码见下方
                JVMUtil.jstack(jStackStream);
            } catch (Throwable t) {
                logger.error("dump jStack error", t);
            } finally {
                guard.release();
            }
            //记录最后一次打印时间
            lastPrintTime = System.currentTimeMillis();
        });
        //must shutdown thread pool ,if not will lead to OOM
        pool.shutdown();//关闭线程池
    }
}

上面代码用到了JVMUtil类,其代码如下:

public class JVMUtil {
    public static void jstack(OutputStream stream) throws Exception {
    	//通过ThreadMXBean获得线程信息
        ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean();
        //dumpAllThreads方法的两个入参true表示需要获取锁定的监视器和锁定的同步器
        for (ThreadInfo threadInfo : threadMxBean.dumpAllThreads(true, true)) {
            stream.write(getThreadDumpString(threadInfo).getBytes());
        }
    }
	//getThreadDumpString定义了堆栈的打印格式,
	//其格式和java自身打印的类似
    private static String getThreadDumpString(ThreadInfo threadInfo) {
        StringBuilder sb = new StringBuilder("\"" + threadInfo.getThreadName() + "\"" +
                " Id=" + threadInfo.getThreadId() + " " +
                threadInfo.getThreadState());
        if (threadInfo.getLockName() != null) {
            sb.append(" on " + threadInfo.getLockName());
        }
        if (threadInfo.getLockOwnerName() != null) {
            sb.append(" owned by \"" + threadInfo.getLockOwnerName() +
                    "\" Id=" + threadInfo.getLockOwnerId());
        }
        if (threadInfo.isSuspended()) {
            sb.append(" (suspended)");
        }
        if (threadInfo.isInNative()) {
            sb.append(" (in native)");
        }
        sb.append('\n');
        int i = 0;

        StackTraceElement[] stackTrace = threadInfo.getStackTrace();
        MonitorInfo[] lockedMonitors = threadInfo.getLockedMonitors();
        for (; i < stackTrace.length && i < 32; i++) {
            StackTraceElement ste = stackTrace[i];
            sb.append("\tat " + ste.toString());
            sb.append('\n');
            if (i == 0 && threadInfo.getLockInfo() != null) {
                Thread.State ts = threadInfo.getThreadState();
                switch (ts) {
                    case BLOCKED:
                        sb.append("\t-  blocked on " + threadInfo.getLockInfo());
                        sb.append('\n');
                        break;
                    case WAITING:
                        sb.append("\t-  waiting on " + threadInfo.getLockInfo());
                        sb.append('\n');
                        break;
                    case TIMED_WAITING:
                        sb.append("\t-  waiting on " + threadInfo.getLockInfo());
                        sb.append('\n');
                        break;
                    default:
                }
            }

            for (MonitorInfo mi : lockedMonitors) {
                if (mi.getLockedStackDepth() == i) {
                    sb.append("\t-  locked " + mi);
                    sb.append('\n');
                }
            }
        }
        if (i < stackTrace.length) {
            sb.append("\t...");
            sb.append('\n');
        }

        LockInfo[] locks = threadInfo.getLockedSynchronizers();
        if (locks.length > 0) {
            sb.append("\n\tNumber of locked synchronizers = " + locks.length);
            sb.append('\n');
            for (LockInfo li : locks) {
                sb.append("\t- " + li);
                sb.append('\n');
            }
        }
        sb.append('\n');
        return sb.toString();
    }
}