java线程池批量导入数据
@Component
public class WifiAndImsiImportTask {
private final int INSERNUM = 50000;//每次添加的条数
private final String FILE_SEPARATOR = System.getProperty("file.separator");
private Logger logger = LoggerFactory.getLogger(getClass());
private String policeStationid;
private String policeAreaid;
private String policeProvinceid;
ExecutorService threadPool = null;//线程池
Map<String, Object> mapImsiLocation=null;
@Resource
WifiAndImsiImportService wifiAndImsiImportService;
@Scheduled(cron = "0 0/5 * * * ?")
public synchronized void runTask() {
if (threadPool == null) {//初始化线程池
threadPool = Executors.newFixedThreadPool(10);
}
policeStationid = SysDictUtils.getDictByCode("policeStationid", "policeStationid");
policeAreaid = SysDictUtils.getDictByCode("policeAreaid", "policeAreaid");
policeProvinceid = SysDictUtils.getDictByCode("provinceid", "provinceid");
logger.info("============================================数据导入开始===================================================");
importData();
logger.info("============================================数据导入结束====================================================");
}
public void importData() {
List<WifiImportEntity> wifiList = new ArrayList<WifiImportEntity>();//存放wifi数据
List<IMSIImportEntity> imsiList = new ArrayList<IMSIImportEntity>();//存放IMSI数据
StringBuffer wifiIds = new StringBuffer();
StringBuffer imsiIds = new StringBuffer();
List<Map<String,Object>> dataList = wifiAndImsiImportService.getWaitImportData();//待导入数据
for (Map<String,Object> map : dataList) {
String ID = "";
try {
ID = map.get("id").toString();
String fileName = map.get("filename").toString();
String filePath = map.get("filepath").toString();
List<String> fileList = FileUtils.readLines(new File(filePath+FILE_SEPARATOR+fileName), "UTF-8");
wifiAndImsiImportService.updateStatus(ID, "1");
if (map.get("filetype")!= null && "1".equals(map.get("filetype"))) {
wifiIds.append(ID).append(",");
wifiImportData(wifiList, fileList, fileName,wifiIds);
} else {
if(mapImsiLocation == null && "3".equals(map.get("filetype"))){
logger.info("==================开始获取归属地============================");
long starttime = System.currentTimeMillis();
mapImsiLocation= wifiAndImsiImportService.getImsiLocation();
logger.info("===耗时："+(System.currentTimeMillis()-starttime)+"毫秒====");
}
imsiIds.append(ID).append(",");
iMSIImportData(imsiList, fileList, fileName,imsiIds,map.get("filetype").toString());
}
} catch(Exception e) {
logger.error("数据导入》》》"+e.toString());
wifiAndImsiImportService.updateStatus(ID, "-1");
}
}
insertSurplusData(wifiList, imsiList, wifiIds, imsiIds);//处理数量不满足的数据
}
/**
* 入库剩余文件
* */
public void insertSurplusData(List<WifiImportEntity> wifiList,List<IMSIImportEntity> imsiList,StringBuffer wifiIds,StringBuffer imsiIds) {
if (wifiList != null && wifiList.size()>0) {
try{
logger.info("============================================WIFI入库开始===================================================");
Future<Boolean> hotWifi = threadPool.submit(new HotWifiImport(wifiList,wifiAndImsiImportService));
Future<Boolean> hotWifihis = threadPool.submit(new HotWifiHisImport(wifiList,wifiAndImsiImportService));
boolean b = hotWifi.get();
boolean b1 = hotWifihis.get();
wifiList.clear();
wifiAndImsiImportService.updateStatus(wifiIds.toString(), "2");
wifiIds.setLength(0);//清空Stringbuffer
logger.info("============================================WIFI入库结束=================================");
} catch(Exception e) {
wifiList.clear();
wifiAndImsiImportService.updateStatus(wifiIds.toString(), "-1");
wifiIds.setLength(0);//清空Stringbuffer
logger.error(e.toString());
}
}
if (imsiList != null && imsiList.size()>0) {
try{
logger.info("============================================IMSI入库开始===================================================");
Future<Boolean> imsiWifi = threadPool.submit(new HotImsiImport(imsiList, wifiAndImsiImportService));
Future<Boolean> imsiWifihis = threadPool.submit(new HotImsiHisImport(imsiList, wifiAndImsiImportService));
boolean b = imsiWifi.get();
boolean b1 = imsiWifihis.get();
imsiList.clear();
wifiAndImsiImportService.updateStatus(imsiIds.toString(), "2");
imsiIds.setLength(0);//清空Stringbuffer
logger.info("============================================IMSI入库结束条数据=================================");
} catch(Exception e) {
imsiList.clear();
wifiAndImsiImportService.updateStatus(imsiIds.toString(), "-1");
imsiIds.setLength(0);//清空Stringbuffer
logger.error(e.toString());
}
}
}
}
//IMSI完整数据入库
class HotImsiHisImport implements Callable<Boolean>{
List<IMSIImportEntity> list = null;
WifiAndImsiImportService wifiAndImsiImportService;
public HotImsiHisImport(List<IMSIImportEntity> list,WifiAndImsiImportService wifiAndImsiImportService) {
this.list = list;
this.wifiAndImsiImportService = wifiAndImsiImportService;
}
@Override
public Boolean call() {
try {
int insertNum = wifiAndImsiImportService.insertIMSIData(list,"social_hot_imeilog_his");
if (insertNum < 0) {
return false;
}
}catch(Exception e) {
return false;
}
return true;
}
}
//IMSI数据入库
class HotImsiImport implements Callable<Boolean>{
private Logger logger = LoggerFactory.getLogger(getClass());
List<IMSIImportEntity> list = null;
WifiAndImsiImportService wifiAndImsiImportService;
private String hotArea;
public HotImsiImport(List<IMSIImportEntity> list,WifiAndImsiImportService wifiAndImsiImportService) {
this.list = list;
this.wifiAndImsiImportService = wifiAndImsiImportService;
hotArea = SysDictUtils.getDictByCode("hot", "hotaera");//热点区域
}
@Override
public Boolean call() {
try {
List<IMSIImportEntity> anasisList = new ArrayList<IMSIImportEntity>();
List<String> imsiRedList = HotRedListTask.getImsiRedList();
for (IMSIImportEntity importEntity : list) {
if (!(imsiRedList.contains(importEntity.getImsi()))) {
String starttime = importEntity.getStarttime();
starttime = starttime.replace("\"", "");
String[] starttimes = starttime.split("\\s+");
String time = starttimes[0];
String hour = starttimes[1].split(":")[0];
String location = importEntity.getLocation();
if (location == null || "".equals(location)) {
location = "其他";
}
importEntity.setGroupBySiteStatic(importEntity.getWsiteid()+"|"+importEntity.getWagentid()+"|"+time);
importEntity.setGroupBySiteTime(importEntity.getWsiteid()+"|"+time+"|"+hour);
importEntity.setGroupBySiteLocation(importEntity.getWsiteid()+"|"+importEntity.getWagentid()+"|"+time+"|"+location);
importEntity.setGroupBySiteTimeIMSI(importEntity.getImsi()+"|"+importEntity.getWsiteid()+"|"+time);
anasisList.add(importEntity);
}
}
anasisData(anasisList);//数据分析
warmResult(anasisList);//预警结果分析
int insertNum = wifiAndImsiImportService.insertIMSIData(anasisList,"social_hot_imeilog");//数据入库
if (insertNum < 0) {
return false;
}
}catch(Exception e) {
return false;
}
return true;
}
//统计IMEI个数
public int getIMEITotal(List<IMSIImportEntity> list) {
int count = 0;
for(IMSIImportEntity iMSIImportEntity:list) {
try {
if (iMSIImportEntity.getImei() != null && !"".equals(iMSIImportEntity.getImei()) && !"000000000000000".equals(iMSIImportEntity.getImei())) {
count++;
}
} catch(Exception e) {
}
}
return count;
}
/**
* 数据分析
* */
public boolean anasisData(List<IMSIImportEntity> anasisList) {
try{
String policeAreaid = SysDictUtils.getDictByCode("policeAreaid", "policeAreaid");
String policeProvinceid = SysDictUtils.getDictByCode("provinceid", "provinceid");
List<Map<String,Object>> updateHotSiteStaticList = new ArrayList<>();//站点时间数量统计-修改
List<Map<String,Object>> updateHotSiteTimeList = new ArrayList<>();//站点时间小时数量统计-修改
List<Map<String,Object>> updateHotSiteLocationList = new ArrayList<>();//站点时间归属地数量统计-修改
List<Map<String,Object>> updateHotSiteIMSITimeList = new ArrayList<>();//IMSI站点时间-修改
List<Map<String,Object>> insertHotSiteStaticList = new ArrayList<>();//站点时间数量统计-添加
List<Map<String,Object>> insertHotSiteTimeList = new ArrayList<>();//站点时间小时数量统计-添加
List<Map<String,Object>> insertHotSiteLocationList = new ArrayList<>();//站点时间归属地数量统计-添加
List<Map<String,Object>> insertHotSiteIMSITimeList = new ArrayList<>();//IMSI站点时间-添加
List<Map<String,Object>> hotSiteStaticList = wifiAndImsiImportService.getSiteStatic();//已存在站点时间
List<Map<String,Object>> hotSiteTimeList = wifiAndImsiImportService.getSiteStaticTime();//已存在站点时间小时
List<Map<String,Object>> hotSiteLocationList = wifiAndImsiImportService.getSiteStaticLocation();//已存在站点时间归属地
List<Map<String,Object>> hotSiteIMSITimeList = wifiAndImsiImportService.getIMSISite();//已存在IMSI站点时间
Map<String, List<IMSIImportEntity>> hotSiteStaticMap = anasisList.stream().collect(Collectors.groupingBy(IMSIImportEntity::getGroupBySiteStatic));
Map<String, List<IMSIImportEntity>> hotSiteTimeMap = anasisList.stream().collect(Collectors.groupingBy(IMSIImportEntity::getGroupBySiteTime));
Map<String, List<IMSIImportEntity>> hotSiteLocationMap = anasisList.stream().collect(Collectors.groupingBy(IMSIImportEntity::getGroupBySiteLocation));
Map<String, List<IMSIImportEntity>> hotSiteIMSITimeMap = anasisList.stream().collect(Collectors.groupingBy(IMSIImportEntity::getGroupBySiteTimeIMSI));
for (String key : hotSiteStaticMap.keySet()) {
try {
Map<String, Object> resultMap = new HashMap<String, Object>();
String[] keys = key.split("\\|");
resultMap.put("SITEID", keys[0]);
resultMap.put("AGENTID", keys[1]);
resultMap.put("STATICTIME", keys[2]);
resultMap.put("PROVINCEID", policeProvinceid);
resultMap.put("AREAID", policeAreaid);
resultMap.put("TYPE", "3");//类型：1：热点wifi,2:嗅探wifi,3:imsi
if (hotSiteStaticList.contains(resultMap)) {//表示数据已存在则修改总数
resultMap.put("TOTAL", hotSiteStaticMap.get(key).size());//不能放在if外边否则会影响判断
updateHotSiteStaticList.add(resultMap);
} else {
resultMap.put("TOTAL", hotSiteStaticMap.get(key).size());
insertHotSiteStaticList.add(resultMap);
}
Map<String, Object> resultIMEIMap = new HashMap<String, Object>();
resultIMEIMap.put("SITEID", keys[0]);
resultIMEIMap.put("AGENTID", keys[1]);
resultIMEIMap.put("STATICTIME", keys[2]);
resultIMEIMap.put("PROVINCEID", policeProvinceid);
resultIMEIMap.put("AREAID", policeAreaid);
resultIMEIMap.put("TYPE", "4");//类型：1：热点wifi,2:嗅探wifi,3:imsi
if (hotSiteStaticList.contains(resultIMEIMap)) {//表示数据已存在则修改总数
resultIMEIMap.put("TOTAL", getIMEITotal(hotSiteStaticMap.get(key)));//不能放在if外边否则会影响判断
updateHotSiteStaticList.add(resultIMEIMap);
} else {
resultIMEIMap.put("TOTAL", getIMEITotal(hotSiteStaticMap.get(key)));
insertHotSiteStaticList.add(resultIMEIMap);
}
} catch(Exception e) {
e.printStackTrace();
}
}
for (String key : hotSiteTimeMap.keySet()) {
try {
Map<String, Object> resulTimetMap = new HashMap<String, Object>();
String[] keys = key.split("\\|");
resulTimetMap.put("SITEID", keys[0]);
resulTimetMap.put("STATICTIME", keys[1]);
resulTimetMap.put("STATICHOUR", keys[2]);
resulTimetMap.put("PROVINCEID", policeProvinceid);
resulTimetMap.put("AREAID", policeAreaid);
resulTimetMap.put("TYPE", "3");//类型：1：热点wifi,2:嗅探wifi,3:imsi
if (hotSiteTimeList.contains(resulTimetMap)) {//表示数据已存在则修改总数
resulTimetMap.put("TOTAL", hotSiteTimeMap.get(key).size());//不能放在if外边否则会影响判断
updateHotSiteTimeList.add(resulTimetMap);
}else {
resulTimetMap.put("TOTAL", hotSiteTimeMap.get(key).size());//不能放在if外边否则会影响判断
insertHotSiteTimeList.add(resulTimetMap);
}
} catch(Exception e) {
e.printStackTrace();
}
}
//importEntity.setGroupBySiteLocation(importEntity.getWsiteid()+"|"+importEntity.getWagentid()+"|"+time+"|"+location);
for (String key : hotSiteLocationMap.keySet()) {
try {
Map<String,Object> resultLocationMap = new HashMap<String,Object>();
String[] keys = key.split("\\|");
resultLocationMap.put("SITEID",keys[0]);
resultLocationMap.put("AGENTID",keys[1]);
resultLocationMap.put("STATICTIME",keys[2]);
String location = keys[3];
String[] locations = locationPattren(location);
resultLocationMap.put("LOCATIONPROVINCENAME", locations[0]);
resultLocationMap.put("LOCATIONAREANAME", locations[1]);
//resultLocationMap.put("TOTAL", hotSiteLocationMap.get(key).size());
resultLocationMap.put("PROVINCEID", policeProvinceid);
resultLocationMap.put("AREAID", policeAreaid);
if (hotSiteLocationList.contains(resultLocationMap)) {//表示数据已存在则修改总数
resultLocationMap.put("TOTAL", hotSiteLocationMap.get(key).size());//不能放在if外边否则会影响判断
updateHotSiteLocationList.add(resultLocationMap);
}else {
resultLocationMap.put("TOTAL", hotSiteLocationMap.get(key).size());//不能放在if外边否则会影响判断
insertHotSiteLocationList.add(resultLocationMap);
}
} catch(Exception e) {
e.printStackTrace();
}
}
if (hotArea.equals("1")) {
logger.info("============================开始遍历IMSI异常站点数据=======================================================================");
long starttime = System.currentTimeMillis();
insertHotSiteIMSITimeList = compareForIMSISite(hotSiteIMSITimeList,hotSiteIMSITimeMap);
logger.info("============================耗时："+(System.currentTimeMillis()-starttime)+"毫秒==========================================");
}
wifiAndImsiImportService.updateSiteStatic(updateHotSiteStaticList);
wifiAndImsiImportService.updateSiteStaticTime(updateHotSiteTimeList);
wifiAndImsiImportService.updateSiteStaticLocation(updateHotSiteLocationList);
wifiAndImsiImportService.updateIMSISite(updateHotSiteIMSITimeList);
wifiAndImsiImportService.insertSiteStatic(insertHotSiteStaticList);
wifiAndImsiImportService.insertSiteStaticTime(insertHotSiteTimeList);
wifiAndImsiImportService.insertSiteStaticLocation(insertHotSiteLocationList);
wifiAndImsiImportService.insertIMSISite(insertHotSiteIMSITimeList);
} catch(Exception e) {
e.printStackTrace();
return false;
}
return true;
}
/**
* 轨迹分析入库比较方法
* */
public List<Map<String,Object>> compareForIMSISite(List<Map<String,Object>> sourceList,Map<String, List<IMSIImportEntity>> hotSiteIMSITimeMap) {
Set<String> dataByFileSet = new HashSet<String>();//文件中获取的数据
Set<String> dataByDataSet = new HashSet<String>();//数据库中获取的数据
Set<String> tempSet = new HashSet<String>();//临时集合
for (String key : hotSiteIMSITimeMap.keySet()) {//Map数据转为set集合
try {
String[] keys = key.split("\\|");
String IMSI = keys[0];
String SITEID = keys[1];
String STATICDAY = keys[2];
String compareValue = IMSI+"|"+SITEID+"|"+STATICDAY;
dataByFileSet.add(compareValue);
} catch(Exception e) {
logger.error(e.toString());
}
}
for (Map<String,Object> map : sourceList) {
String compareValue = map.get("COMPAREVALUE").toString();
dataByDataSet.add(compareValue);
}
tempSet.addAll(dataByFileSet);
tempSet.removeAll(dataByDataSet);
List<Map<String,Object>> list = new ArrayList<>();//IMSI站点时间-修改
for (String s : tempSet) {//转为list集合
try{
String[] ss = s.split("\\|");
Map<String,Object> params = new HashMap<String,Object>();
params.put("IMSI", ss[0]);
params.put("SITEID", ss[1]);
params.put("STATICDAY", ss[2]);
list.add(params);
}catch(Exception e) {
logger.error(e.toString());
}
}
return list;
}
public String[] locationPattren(String location) {
String[] result = new String[2];
if (location == null || "".equals(location)) {
result[0] = "其他";
result[1] = "其他";
} else if(Pattern.matches(".*-.*", location)){
result[0] = location.split("-")[0];
result[1] = location.split("-")[1];
} else if(Pattern.matches(".*省.*市", location)){
result[0] = location.split("省")[0];
String s = location.split("省")[1];
result[1] = s.substring(0, s.length()-1);
} else if(Pattern.matches(".*省.*", location)){
result[0] = location.split("省")[0];
result[1] = location.split("省")[1];
} else if(Pattern.matches(".*市.*", location)){
if (location.split("市").length > 1) {
result[0] = location.split("市")[0];
result[1] = location.split("市")[1];
} else {
result[0] = location.split("市")[0];
result[1] = "其他";
}
} else {
List<String> plist = new ArrayList<String>();
plist.add("内蒙古");
plist.add("广西");
plist.add("西藏");
plist.add("宁夏");
plist.add("新疆");
boolean isempty = true;
for (String p : plist) {
if (location.contains(p)) {
result[0] = p;
result[1] = location.substring(p.length(), location.length());
isempty = false;
}
}
if (isempty) {
result[0] = location;
result[1] = "其他";
}
}
return result;
}
/**
* 获取符合预警值得结果
* */
public void warmResult(List<IMSIImportEntity> anasisList) {
try {
List<HotWarmImportEntity> warmList = wifiAndImsiImportService.getWarmInfo("3");//3:imsi实时预警，4：wifi实时预警5:imei实时预警
List<Map<String,Object>> resultList = new ArrayList<Map<String,Object>>();
for (IMSIImportEntity imsiImportEntity : anasisList) {
try{
for (HotWarmImportEntity hotWarmEntity : warmList) {
if (hotWarmEntity.getWarmvalue().equals(imsiImportEntity.getImsi()) || hotWarmEntity.getWarmvalue().equals(imsiImportEntity.getImei())) {
Map<String,Object> map = new HashMap<String,Object>();
map.put("id", hotWarmEntity.getId());
map.put("warmTime", imsiImportEntity.getStarttime());
map.put("siteId", imsiImportEntity.getWsiteid());
resultList.add(map);
}
}
} catch(Exception e) {
e.printStackTrace();
}
}
wifiAndImsiImportService.insertWarmResult(resultList);
} catch(Exception e) {
e.printStackTrace();
}
}
}
本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。
