本文基于dubbo 2.7.5版本代码
在一些场景中,比如服务端收到请求需要在业务线程池中处理请求时,dubbo需要通过ExecutorRepository创建线程池。在创建线程池的时候,dubbo设置了RejectedExecutionHandler,也就是当线程池满的时候,会调用RejectedExecutionHandler。dubbo提供了RejectedExecutionHandler的实现类AbortPolicyWithReport。当线程池满的时候,AbortPolicyWithReport将线程的堆栈信息打印到执行的文件中。下面来介绍一下AbortPolicyWithReport的实现。
在dubbo中可以通过dump.directory指定打印堆栈信息的目录,如果不设置,dubbo将堆栈信息打印到System.getProperty(“user.home”)目录下。设置dump.directory的方式可以有:
- @Service(parameters = {“dump.directory”,"/tmp"}),这个设置只对单个的服务生效
- dubbo.application.parameters[dump.directory]=/tmp,这样设置会使整个应用的所有服务都生效
下面是AbortPolicyWithReport源码。
public class AbortPolicyWithReport extends ThreadPoolExecutor.AbortPolicy {
protected static final Logger logger = LoggerFactory.getLogger(AbortPolicyWithReport.class);
//threadName可以作为日志打印的关键字,默认是Dubbo,可以通过参数“threadname”设置
private final String threadName;
//url是由服务参数组成的,
//如果是服务端创建AbortPolicyWithReport ,则url是服务端发布服务的url
//如果是消费端,则url代表了消费端需要引用的远程服务url
private final URL url;
//表示最后一次打印堆栈信息的事件,dubbo每10分钟打印一次
private static volatile long lastPrintTime = 0;
//表示堆栈信息的打印间隔
private static final long TEN_MINUTES_MILLS = 10 * 60 * 1000;
private static final String OS_WIN_PREFIX = "win";
private static final String OS_NAME_KEY = "os.name";
private static final String WIN_DATETIME_FORMAT = "yyyy-MM-dd_HH-mm-ss";
private static final String DEFAULT_DATETIME_FORMAT = "yyyy-MM-dd_HH:mm:ss";
private static Semaphore guard = new Semaphore(1);
public AbortPolicyWithReport(String threadName, URL url) {
this.threadName = threadName;
this.url = url;
}
@Override
public void rejectedExecution(Runnable r, ThreadPoolExecutor e) {
String msg = String.format("Thread pool is EXHAUSTED!" +
" Thread Name: %s, Pool Size: %d (active: %d, core: %d, max: %d, largest: %d), Task: %d (completed: "
+ "%d)," +
" Executor status:(isShutdown:%s, isTerminated:%s, isTerminating:%s), in %s://%s:%d!",
threadName, e.getPoolSize(), e.getActiveCount(), e.getCorePoolSize(), e.getMaximumPoolSize(),
e.getLargestPoolSize(),
e.getTaskCount(), e.getCompletedTaskCount(), e.isShutdown(), e.isTerminated(), e.isTerminating(),
url.getProtocol(), url.getIp(), url.getPort());
//打印日志,其中属性theadName会打印到日志中
logger.warn(msg);
//调用打印堆栈信息的方法
dumpJStack();
throw new RejectedExecutionException(msg);
}
private void dumpJStack() {
long now = System.currentTimeMillis();
//一个线程池每10分钟打印一次堆栈信息
if (now - lastPrintTime < TEN_MINUTES_MILLS) {
return;
}
//guard是信号量,同时只能一个线程打印堆栈
if (!guard.tryAcquire()) {
return;
}
ExecutorService pool = Executors.newSingleThreadExecutor();
//使用java原生线程池打印堆栈信息,这个线程池中只有一个线程,用完就关闭
pool.execute(() -> {
//获得的打印堆栈信息文件的目录
String dumpPath = url.getParameter(DUMP_DIRECTORY, System.getProperty("user.home"));
SimpleDateFormat sdf;
String os = System.getProperty(OS_NAME_KEY).toLowerCase();
// 不同系统的时间格式不同
if (os.contains(OS_WIN_PREFIX)) {
sdf = new SimpleDateFormat(WIN_DATETIME_FORMAT);
} else {
sdf = new SimpleDateFormat(DEFAULT_DATETIME_FORMAT);
}
String dateStr = sdf.format(new Date());
//try-with-resources
//打印的文件名为:Dubbo_JStack.log.时间,其中时间到秒
try (FileOutputStream jStackStream = new FileOutputStream(
new File(dumpPath, "Dubbo_JStack.log" + "." + dateStr))) {
//使用JVMUtil打印,代码见下方
JVMUtil.jstack(jStackStream);
} catch (Throwable t) {
logger.error("dump jStack error", t);
} finally {
guard.release();
}
//记录最后一次打印时间
lastPrintTime = System.currentTimeMillis();
});
//must shutdown thread pool ,if not will lead to OOM
pool.shutdown();//关闭线程池
}
}
上面代码用到了JVMUtil类,其代码如下:
public class JVMUtil {
public static void jstack(OutputStream stream) throws Exception {
//通过ThreadMXBean获得线程信息
ThreadMXBean threadMxBean = ManagementFactory.getThreadMXBean();
//dumpAllThreads方法的两个入参true表示需要获取锁定的监视器和锁定的同步器
for (ThreadInfo threadInfo : threadMxBean.dumpAllThreads(true, true)) {
stream.write(getThreadDumpString(threadInfo).getBytes());
}
}
//getThreadDumpString定义了堆栈的打印格式,
//其格式和java自身打印的类似
private static String getThreadDumpString(ThreadInfo threadInfo) {
StringBuilder sb = new StringBuilder("\"" + threadInfo.getThreadName() + "\"" +
" Id=" + threadInfo.getThreadId() + " " +
threadInfo.getThreadState());
if (threadInfo.getLockName() != null) {
sb.append(" on " + threadInfo.getLockName());
}
if (threadInfo.getLockOwnerName() != null) {
sb.append(" owned by \"" + threadInfo.getLockOwnerName() +
"\" Id=" + threadInfo.getLockOwnerId());
}
if (threadInfo.isSuspended()) {
sb.append(" (suspended)");
}
if (threadInfo.isInNative()) {
sb.append(" (in native)");
}
sb.append('\n');
int i = 0;
StackTraceElement[] stackTrace = threadInfo.getStackTrace();
MonitorInfo[] lockedMonitors = threadInfo.getLockedMonitors();
for (; i < stackTrace.length && i < 32; i++) {
StackTraceElement ste = stackTrace[i];
sb.append("\tat " + ste.toString());
sb.append('\n');
if (i == 0 && threadInfo.getLockInfo() != null) {
Thread.State ts = threadInfo.getThreadState();
switch (ts) {
case BLOCKED:
sb.append("\t- blocked on " + threadInfo.getLockInfo());
sb.append('\n');
break;
case WAITING:
sb.append("\t- waiting on " + threadInfo.getLockInfo());
sb.append('\n');
break;
case TIMED_WAITING:
sb.append("\t- waiting on " + threadInfo.getLockInfo());
sb.append('\n');
break;
default:
}
}
for (MonitorInfo mi : lockedMonitors) {
if (mi.getLockedStackDepth() == i) {
sb.append("\t- locked " + mi);
sb.append('\n');
}
}
}
if (i < stackTrace.length) {
sb.append("\t...");
sb.append('\n');
}
LockInfo[] locks = threadInfo.getLockedSynchronizers();
if (locks.length > 0) {
sb.append("\n\tNumber of locked synchronizers = " + locks.length);
sb.append('\n');
for (LockInfo li : locks) {
sb.append("\t- " + li);
sb.append('\n');
}
}
sb.append('\n');
return sb.toString();
}
}