1.线程池工具
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import java.util.concurrent.CompletableFuture;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.Executor;
import java.util.concurrent.ForkJoinPool;
import java.util.concurrent.atomic.AtomicBoolean;
import java.util.concurrent.atomic.AtomicInteger;
import java.util.function.Function;
/**
* @author: X-Man
* @date: 2022/2/11 15:23
* @desc: 任务分解多线程运行,同步等待工具类
*/
public class TaskDivRunSyncWaitUtil {
private static final int TEN = 10;
private static final int HUNDRED = 100;
private static final int THOUSAND = 1000;
private static final Logger LOGGER = LoggerFactory.getLogger(TaskDivRunSyncWaitUtil.class);
public static int exe(String taskName, Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc, int dataTotalSize) {
return exeBody(taskName, asyncBzFunc, dataTotalSize, HUNDRED, 2, 1);
}
public static int exeFromPage2(String taskName, Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc, int dataTotalSize) {
return exeBody(taskName, asyncBzFunc, dataTotalSize, HUNDRED, 2, 2);
}
public static int exe(String taskName, Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc, int dataTotalSize, int batchSize) {
return exeBody(taskName, asyncBzFunc, dataTotalSize, batchSize, 2, 1);
}
public static int exeFromPage2(String taskName, Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc, int dataTotalSize, int batchSize) {
return exeBody(taskName, asyncBzFunc, dataTotalSize, batchSize, 2, 2);
}
public static int exe(String taskName, Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc, int dataTotalSize, int batchSize, int cpuTimes) {
return exeBody(taskName, asyncBzFunc, dataTotalSize, batchSize, cpuTimes, 1);
}
public static int exeFromPage2(String taskName, Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc, int dataTotalSize, int batchSize, int cpuTimes) {
return exeBody(taskName, asyncBzFunc, dataTotalSize, batchSize, cpuTimes, 2);
}
/**
* 指定线程池线程数量,多线程分片执行任务,同步等待
* 支持某一分片执行完成后标记停止(前置性校验,非线程中断)其它或后续分片任务执行
*
* @param taskName 任务名称,通常由id和名称构成,用以区分并发任务日志
* @param asyncBzFunc 业务func,其中,线程相关参数,如起始下标、结束下标、线程名称等将在本方法中通过TaskDivRunThreadParam参数回传
* @param dataTotalSize 业务处理数据总数
* @param batchSize 分批处理业务时,每批次处理的数据量
* @param cpuTimes 线程池的大小/cpu数量的倍数
* @param fromPageNum 从分页第几页(起始值默认1)开始执行任务,适用于分页查询等需要在第一次业务处理时获取到分页参数的任务的后续分页多线程处理
* @return
*/
private static int exeBody(String taskName, Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc, int dataTotalSize, int batchSize, int cpuTimes, int fromPageNum) {
if (dataTotalSize < 2) {
return -1;
}
if (cpuTimes < 1 || cpuTimes > TEN) {
cpuTimes = 2;
}
if (batchSize < TEN || batchSize > THOUSAND) {
batchSize = 2 * HUNDRED;
}
int startIndex = 0;
AtomicInteger bzProcessSuccessCount = new AtomicInteger(0);
int needThreadCount = dataTotalSize / batchSize + (dataTotalSize % batchSize == 0 ? 0 : 1);
if (fromPageNum > 1) {
LOGGER.debug("{}业务从第{}页开始执行", taskName, fromPageNum);
needThreadCount -= (fromPageNum - 1);
if (needThreadCount < 1) {
return -1;
}
startIndex += (fromPageNum - 1) * batchSize;
}
/**
* 自定义线程池:线程数量等于cpu数量的指定倍数
* 对于IO密集型业务,倍数建议2以上
* 对于CPU密集型业务,倍数建议为1
*/
int realThreadCount = Math.min(needThreadCount, cpuTimes * Runtime.getRuntime().availableProcessors());
Executor executor = new ForkJoinPool(realThreadCount);
/**
* 闭锁计数:业务拆分数量
*/
CountDownLatch downLatch = new CountDownLatch(needThreadCount);
int finalBatchSize = batchSize;
int cycleIndex = 0;
if (fromPageNum > 1) {
cycleIndex += (fromPageNum - 1);
}
long startTimeOuter = System.currentTimeMillis();
LOGGER.debug(taskName + "---start业务执行---:threadPoolSize={},cycleSize={},dataTotalSize={}",
realThreadCount, needThreadCount, dataTotalSize);
AtomicBoolean isStopAllAfterTask = new AtomicBoolean(false);
AtomicBoolean isStopAllOtherTask = new AtomicBoolean(false);
AtomicInteger stopCycleIndex = new AtomicInteger(-1);
AtomicInteger realRunTaskCount = new AtomicInteger(needThreadCount);
for (; startIndex < dataTotalSize; startIndex += batchSize) {
int finalCycleIndex = cycleIndex;
Function<Integer, Void> asyncProcessFunc = startIndexCp -> {
/**
* 获取业务拆分下标参数:startIndex、endIndex
*/
int endIndexCp = startIndexCp + finalBatchSize;
if (endIndexCp > dataTotalSize) {
endIndexCp = dataTotalSize;
}
/**
* 如果需要停止所有其它任务
*/
if (isStopAllOtherTask.get()) {
LOGGER.debug(taskName + "stop all other task业务拆分执行[{}] because业务分片[{}]:threadPoolSize={},threadName={},cycleIndex={},startIndex={},endIndex={}",
finalCycleIndex, stopCycleIndex.get(), realThreadCount, Thread.currentThread().getName(), finalCycleIndex, startIndexCp, endIndexCp);
realRunTaskCount.decrementAndGet();
return null;
}
/**
* 如果需要停止后续所有任务
*/
if (isStopAllAfterTask.get() && finalCycleIndex > stopCycleIndex.get()) {
LOGGER.debug(taskName + "stop all after task业务拆分执行[{}] because业务分片[{}]:threadPoolSize={},threadName={},cycleIndex={},startIndex={},endIndex={}",
finalCycleIndex, stopCycleIndex.get(), realThreadCount, Thread.currentThread().getName(), finalCycleIndex, startIndexCp, endIndexCp);
realRunTaskCount.decrementAndGet();
return null;
}
/**
* 执行业务
*/
long startTime = System.currentTimeMillis();
LOGGER.debug(taskName + "start业务拆分执行[{}]:threadPoolSize={},threadName={},cycleIndex={},startIndex={},endIndex={}",
finalCycleIndex, realThreadCount, Thread.currentThread().getName(), finalCycleIndex, startIndexCp, endIndexCp);
TaskDivRunThreadParam param = new TaskDivRunThreadParam();
param.setStartIndex(startIndexCp);
param.setEndIndex(endIndexCp);
param.setPageSize(finalBatchSize);
param.setPageNum(finalCycleIndex + 1);
param.setThreadName(Thread.currentThread().getName());
param.setThreadPoolSize(realThreadCount);
TaskDivRunThreadResult threadResult = asyncBzFunc.apply(param);
if (null == threadResult) {
threadResult = new TaskDivRunThreadResult();
}
if (threadResult.isStopAllAfterTask() && !isStopAllAfterTask.get()) {
isStopAllAfterTask.set(true);
stopCycleIndex.set(finalCycleIndex);
}
if (threadResult.isStopAllOtherTask() && !isStopAllOtherTask.get()) {
isStopAllOtherTask.set(true);
stopCycleIndex.set(finalCycleIndex);
}
Integer successCount = threadResult.getSuccessCount();
long endTime = System.currentTimeMillis();
LOGGER.debug(taskName + "end业务拆分执行[{}]:costSeconds={}", finalCycleIndex, ((endTime - startTime) / 1000 + "." + (endTime - startTime) % 1000));
/**
* 成功数量计数
*/
bzProcessSuccessCount.addAndGet(null == successCount ? 0 : successCount);
return null;
};
final int finalStartIndex = startIndex;
/**
* 异步线程并发执行业务
*/
CompletableFuture.supplyAsync(() -> asyncProcessFunc.apply(finalStartIndex), executor).whenComplete((r, e) -> {
/**
* 异常处理和减锁
*/
if (null != e) {
LOGGER.error("", e);
}
downLatch.countDown();
LOGGER.debug(taskName + "milestone:剩余未完成任务数={}", downLatch.getCount());
});
if (startIndex < dataTotalSize) {
cycleIndex++;
}
}
/**
* 闭锁等待
*/
try {
LOGGER.debug(taskName + "---wait业务执行---");
downLatch.await();
} catch (InterruptedException e) {
LOGGER.error("", e);
}
long endTimeOuter = System.currentTimeMillis();
LOGGER.debug(taskName + "---end业务执行---:divTotalTaskCount={},realRunTaskCount={},costSeconds={},dataTotalSize={},successDataCount={}",
needThreadCount, realRunTaskCount.get(),
((endTimeOuter - startTimeOuter) / 1000 + "." + (endTimeOuter - startTimeOuter) % 1000),
dataTotalSize, bzProcessSuccessCount.get());
return bzProcessSuccessCount.get();
}
}
2.业务回调函数入参
/**
* @author: X-Man
* @date: 2022/2/11 15:43
* @desc: 批量业务回调函数入参
*/
public class TaskDivRunThreadParam {
/**
* 其起始下标(从0开始),include
*/
private int startIndex;
/**
* 截至下标,exclude
*/
private int endIndex;
/**
* 分页大小
*/
private int pageSize;
/**
* 分页当前页码(从1开始)
*/
private int pageNum;
/**
* 线程池大小
*/
private int threadPoolSize;
/**
* 当前线程名称
*/
private String threadName;
public TaskDivRunThreadParam() {
}
public TaskDivRunThreadParam(int startIndex, int endIndex) {
this.startIndex = startIndex;
this.endIndex = endIndex;
}
public TaskDivRunThreadParam(int startIndex, int endIndex, int threadPoolSize, String threadName) {
this.startIndex = startIndex;
this.endIndex = endIndex;
this.threadPoolSize = threadPoolSize;
this.threadName = threadName;
}
public int getStartIndex() {
return startIndex;
}
public void setStartIndex(int startIndex) {
this.startIndex = startIndex;
}
public int getEndIndex() {
return endIndex;
}
public void setEndIndex(int endIndex) {
this.endIndex = endIndex;
}
public int getThreadPoolSize() {
return threadPoolSize;
}
public void setThreadPoolSize(int threadPoolSize) {
this.threadPoolSize = threadPoolSize;
}
public String getThreadName() {
return threadName;
}
public void setThreadName(String threadName) {
this.threadName = threadName;
}
public int getPageSize() {
return pageSize;
}
public void setPageSize(int pageSize) {
this.pageSize = pageSize;
}
public int getPageNum() {
return pageNum;
}
public void setPageNum(int pageNum) {
this.pageNum = pageNum;
}
}
3.业务回调函数出参
/**
* @author: X-Man
* @date: 2022/2/11 18:14
* @desc: 批量业务回调函数出参
*/
public class TaskDivRunThreadResult {
private int successCount;
/**
* 是否停止所有其他任务:适用于findOne类型任务
*/
private boolean stopAllOtherTask;
/**
* 是否停止所有后续下标任务:适用于findFirst类型任务
*/
private boolean stopAllAfterTask;
/**
* 当前业务分片下标
*/
private int currentCycleIndex;
public TaskDivRunThreadResult() {
}
public TaskDivRunThreadResult(int successCount) {
this.successCount = successCount;
}
public TaskDivRunThreadResult(int successCount, boolean stopAllOtherTask, boolean stopAllAfterTask) {
this.successCount = successCount;
this.stopAllOtherTask = stopAllOtherTask;
this.stopAllAfterTask = stopAllAfterTask;
}
public int getSuccessCount() {
return successCount;
}
public void setSuccessCount(int successCount) {
this.successCount = successCount;
}
public boolean isStopAllOtherTask() {
return stopAllOtherTask;
}
public void setStopAllOtherTask(boolean stopAllOtherTask) {
this.stopAllOtherTask = stopAllOtherTask;
}
public boolean isStopAllAfterTask() {
return stopAllAfterTask;
}
public void setStopAllAfterTask(boolean stopAllAfterTask) {
this.stopAllAfterTask = stopAllAfterTask;
}
public int getCurrentCycleIndex() {
return currentCycleIndex;
}
public void setCurrentCycleIndex(int currentCycleIndex) {
this.currentCycleIndex = currentCycleIndex;
}
}
4.Demo:多线程查询1万条数据
int total = 10000;
List<XXX> resultData = new ArrayList<XXX>();
Function<TaskDivRunThreadParam, TaskDivRunThreadResult> asyncBzFunc = param -> {
PageHelper.startPage(param.getPageNum(), pageSize);
List<XXX> listAsync = 数据查询
if (!CollectionUtils.isEmpty(listAsync)) {
synchronized (resultData) {
resultData.addAll(listAsync);
}
}
return null;
};
TaskDivRunSyncWaitUtil.exe("<" + 任务id + "@任务名称>", asyncBzFunc, total, pageSize);