1 package com.test.pic.crawler;
  2 
  3 import java.io.File;
  4 import java.io.FileOutputStream;
  5 import java.io.IOException;
  6 import java.io.InputStream;
  7 import java.io.OutputStream;
  8 import java.net.URL;
  9 import java.net.URLConnection;
 10 import java.util.Arrays;
 11 import java.util.HashSet;
 12 import java.util.List;
 13 import java.util.Set;
 14 import java.util.concurrent.BlockingQueue;
 15 import java.util.concurrent.LinkedBlockingDeque;
 16 import java.util.concurrent.ScheduledExecutorService;
 17 import java.util.concurrent.ScheduledThreadPoolExecutor;
 18 import org.apache.commons.lang3.concurrent.BasicThreadFactory;
 19 import org.jsoup.Jsoup;
 20 import org.jsoup.nodes.*;
 21 import org.jsoup.select.Elements;
 22 
 23 
 24 
 25 
 26 /**   
 27  * @Title: PicCrawler.java 
 28  *
 29  * @Package com.test.pic.crawler 
 30  *
 31  * @Description: 爬取指定网站的指定Tag下的图片或者全部Tag图片 
 32  *
 33  * @author CoderZZ   
 34  *
 35  * @date 2018年1月12日 下午11:22:41 
 36  *
 37  * @version V1.0   
 38  *
 39  */
 40 public class PicCrawler implements Runnable{
 41     private static String pathString = "G:/test/pic/";//存储目录
 42     //存储真正的爬取页面
 43     static BlockingQueue<String> urlBlockingQueue = new LinkedBlockingDeque<String>(1000);
 44     static int threadNum = 10;
 45 //    public PicCrawler(String url){
 46 //        this.url = url;
 47 //    }
 48 
 49     /** 
 50      * @Title: main 
 51      *
 52      * @Description: TODO(这里用一句话描述这个方法的作用) 
 53      *
 54      * @param @param args    设定文件 
 55      *
 56      * @return void    返回类型 
 57      *
 58      * @throws 
 59      *
 60      */
 61     public static void main(String[] args) {
 62         String homeurlString = "https://www.xxxx.com";//爬取页面的基本地址
 63         String tagPageUrl = "https://www.xxxx.com/tag/";//tag分页地址
 64         //Tag标签的完整路径
 65         Set<String> tagFullHrefSet = new HashSet<String>(16);
 66         //想要爬取哪些tag，如果为空，则全部爬取;否则只配置对应的tag
 67         String[] crawlerTagArray = {"风景"};
 68         List<String> crawlerTagList = Arrays.asList(crawlerTagArray);
 69         try {
 70             //1.获取想要的tag完整的url
 71             Document tagListDocument = Jsoup.connect(tagPageUrl).get();
 72             Elements tagsListDivElements = tagListDocument.getElementsByClass("tags_list");
 73             for(Element element:tagsListDivElements){
 74                 Elements aElements = element.getElementsByTag("a");
 75                 for(Element a:aElements){
 76                     if(crawlerTagList.size() == 0 || crawlerTagList.contains(a.text())){
 77                         String tagUrlString = homeurlString+a.attr("href");
 78                         //https://www.xxxx.com/tag/fengjing.html
 79                         tagUrlString = tagUrlString.substring(0, tagUrlString.lastIndexOf("."))+"/1.html";
 80                         tagFullHrefSet.add(tagUrlString);
 81                     }
 82                 }
 83             }
 84             //2.获取图片链接页面地址，分页爬取
 85             for(String tagUrl:tagFullHrefSet){
 86                 String tempTagUrlString = tagUrl;
 87                 int currentPageNum = 1;
 88                 while(true){
 89                     try{
 90                         Document imagePageDocument = Jsoup.connect(tempTagUrlString).get();
 91                         Elements imageListElements = imagePageDocument.getElementsByClass("Pli-litpic");
 92                         if(imageListElements.size() == 0){
 93                             break;
 94                         }
 95                         for(Element image:imageListElements){
 96                             urlBlockingQueue.offer(homeurlString+image.attr("href"));
 97                         }
 98                         //https://www.xxxx.com/tag/fengjing/1.html
 99                         tempTagUrlString = tempTagUrlString.substring(0, tempTagUrlString.lastIndexOf("/")+1)+(++currentPageNum)+".html";
100                     }catch(Exception e){
101                         break;
102                     }
103                 }
104             }
105             ScheduledExecutorService excutor = new ScheduledThreadPoolExecutor(threadNum,new BasicThreadFactory.Builder().namingPattern("my-crawler-thread-%d").daemon(false).build());
106             for(int i=0;i<threadNum;i++){
107 //                excutor.schedule(new PicCrawler(urlArray[i]), 1, TimeUnit.SECONDS);
108 //                excutor.execute(new PicCrawler(urlArray[i]));
109                 excutor.submit(new PicCrawler());
110             }
111         } catch (IOException e) {
112             // TODO Auto-generated catch block
113             e.printStackTrace();
114         }
115     }
116     @Override
117     public void run() {
118         while (true) {
119             try {
120                 long begin = System.currentTimeMillis();
121                 String url = urlBlockingQueue.poll();
122                 if(null != url){
123                     Document doc = Jsoup.connect(url).get();
124                     Elements titleElements =doc.select("#photos > h1");
125                     if(null != titleElements && null != titleElements.get(0)){
126                         Set<String> imgSrcSet = new HashSet<String>(16);
127                         Element titleElement = titleElements.get(0);
128                         String foldNameString = titleElement.text();
129                         String[] nameArray = foldNameString.split("\\(");
130                         foldNameString = nameArray[0];
131                         nameArray = nameArray[1].split("/");
132                         int totalPaggs = Integer.parseInt(nameArray[1].replace(")", ""));
133                         for(int i=1;i<=totalPaggs;i++){
134                             String urlTemp = url.replace(".html", "_"+i+".html");
135                             Document docTemp = Jsoup.connect(urlTemp).get();
136                             Element element = docTemp.getElementById("big-pic");
137                             Elements imgElements = element.getElementsByTag("img");
138                             for(Element imgElement:imgElements){
139                                 imgSrcSet.add(imgElement.attr("src"));
140                             }
141                         }
142                         if(imgSrcSet.size()>0){
143                             for(String imgSrc:imgSrcSet){
144                                 // 构造URL    
145                                 URL imgurl = new URL(imgSrc);    
146                                 // 打开连接    
147                                 URLConnection con = imgurl.openConnection();    
148                                 //设置请求超时为10s    
149                                 con.setConnectTimeout(10*1000);    
150                                 // 输入流    
151                                 InputStream is = con.getInputStream();    
152                                 // 500k的数据缓冲    
153                                 byte[] bs = new byte[1024*500];    
154                                 // 读取到的数据长度    
155                                 int len;    
156                                 // 输出的文件流    
157                                 File sf=new File(pathString+"\\"+foldNameString);    
158                                 if(!sf.exists()){    
159                                     sf.mkdirs();    
160                                 }
161                                 String filename = imgSrc.split("/")[imgSrc.split("/").length-1];
162                                 OutputStream os = new FileOutputStream(sf.getPath()+"\\"+filename);    
163                                 // 开始读取    
164                                 while ((len = is.read(bs)) != -1) {    
165                                     os.write(bs, 0, len);    
166                                 }    
167                                 // 完毕，关闭所有链接    
168                                 os.close();    
169                                 is.close();  
170                                 System.out.println(imgSrc+"下载完成!!!");
171                             }
172                         }
173                         long end = System.currentTimeMillis();
174                         System.out.println("================================================================");
175                         System.out.println(Thread.currentThread().getName()+"******************已全部下载完成，用时："+((end-begin)/1000)+"S");
176                     }
177                 }else{
178                     System.out.println("========================BlockingQueue已空，已全部抓取完成!=======================");
179                 }
180             } catch (Exception e) {
181                 System.out.println("========================抓取异常=======================");
182             }
183         }
184     }
185 }