java 获取远程文件批量下载 java多线程批量下载文件

转载

技术极客侠 2024-03-03 10:41:03

文章标签 java 获取远程文件批量下载 java runtime 文件路径 .net 文章分类 Java 后端开发

一. 多线程下载文件考虑处理步骤：

1. 如何获取文件的长度

2. 合理的创建线程数量，并计算每一个线程下载的长度

3. 如何将多个线程下载的字节写入到文件中

二. 代码实现如下：

package com.bochao.download;

import java.io.File;
import java.net.URL;
import java.net.URLConnection;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;

/**
 * HTTP多线程下载
 * 
 * @Author DuanCZ
 * @Date 2015年9月30日-上午9:28:28
 */

public class HttpMulitThreadDownload {

	// 下载文件路径
	private String downloadFilePath = null;
	// 保存文件路径
	private String saveFileDir = "c:\\";
	// 默认合理并发线程数
	private int threadCount = Runtime.getRuntime().availableProcessors() * 2;
	// 新文件名称
	private String newFileName = null;

	public HttpMulitThreadDownload(int threadCount, String downloadFilePath, String saveFileDir, String newFileName) {
		// 用户指定线程数如果小于默认线程数则使用用户指定线程数
		if (threadCount < this.threadCount) {
			this.threadCount = threadCount;
		}
		this.downloadFilePath = downloadFilePath;
		this.saveFileDir = saveFileDir;
		this.newFileName = newFileName;
	}

	public void MulitThreadDownload() {

		// 数据合法性验证
		if (null == downloadFilePath || downloadFilePath.isEmpty()) {
			throw new RuntimeException("请指定下载路径!");
		}
		if (null == saveFileDir || saveFileDir.isEmpty()) {
			throw new RuntimeException("请指定保存路径!");
		}

		// 创建保存文件路径如果不存在
		File saveFileDirTemp = new File(saveFileDir);
		if (!saveFileDirTemp.exists()) {
			saveFileDirTemp.mkdirs();
		}

		// 处理文件名称
		if (null == newFileName || newFileName.isEmpty()) {
			newFileName = downloadFilePath.substring(downloadFilePath.lastIndexOf("/") + 1, downloadFilePath.length());
		}

		// 创建固定大小的线程池
		ExecutorService threadPool = Executors.newFixedThreadPool(threadCount);
		try {
			// 根据文件长度计算合理线程数开始
			URLConnection urlConnection = new URL(downloadFilePath).openConnection();
			// 获取文件长度
			int downloadFileLength = urlConnection.getContentLength();
			// 计算每个线程负责的文件字节长度
			int averageThreadLength = downloadFileLength / threadCount;
			int residueThreadLength = downloadFileLength % threadCount;
			// 让每一个线程开始工作
			int startIndex = 0;
			int endIndex = 0;
			for (int i = 0; i < threadCount; i++) {

				// 计算每一个线程开始和计数索引
				startIndex = i * averageThreadLength;
				// 如果是最后一个线程，则将剩余的全部下载
				if ((i + 1) == threadCount) {
					endIndex = (i + 1) * averageThreadLength + residueThreadLength - 1;
				}
				endIndex = (i + 1) * averageThreadLength - 1;

				// 创建下载线程对象
				DownloadHandlerThread downloadHandlerThread = new DownloadHandlerThread();
				downloadHandlerThread.setDownloadFilePath(downloadFilePath);
				downloadHandlerThread.setSaveFilePath(saveFileDir + newFileName);
				downloadHandlerThread.setStartIndex(startIndex);
				downloadHandlerThread.setEndIndex(endIndex);
				threadPool.execute(downloadHandlerThread);
			}
		} catch (Exception e) {
			e.printStackTrace();
		} finally {
			// 关闭线程池
			threadPool.shutdown();
		}
	}
}

package com.bochao.download;

import java.io.IOException;
import java.io.InputStream;
import java.io.RandomAccessFile;
import java.net.URL;
import java.net.URLConnection;
import java.text.SimpleDateFormat;
import java.util.Date;

/**
 * 下载文件处理线程
 * 
 * @Author DuanCZ
 * @Date 2015年9月30日-上午11:38:42
 */

public class DownloadHandlerThread implements Runnable {

    // 待下载的HTTP文件路径
    private String downloadFilePath = null;
    // 下载保存文件路径
    private String saveFilePath = null;
    // 文件随机写入开始索引
    private int startIndex = 0;
    // 文件随机写入结束索引
    private int endIndex = 0;

    @Override
    public void run() {

        System.out.println("线程名称[" + Thread.currentThread().getName() + "]于时间["
                + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "]开始下载....");

        // 文件输入流对象
        InputStream fileInputStream = null;
        // 随机访问文件
        RandomAccessFile randomAccessFile = null;

        // ------------------------------------------------------------------------------
        // 获取随机文件流开始
        // 获取一个URL打开链接对象
        URLConnection urlConnection = null;
        try {
            urlConnection = new URL(downloadFilePath).openConnection();
        } catch (IOException e1) {
            e1.printStackTrace();
        }
        // 设置该链接允许和用户交互
        urlConnection.setAllowUserInteraction(true);
        // 设置请求属性字节范围
        urlConnection.setRequestProperty("Range", "bytes=" + startIndex + "-" + endIndex + "");
        try {
            // 获取指定的文件流
            fileInputStream = urlConnection.getInputStream();
        } catch (IOException e) {
            e.printStackTrace();
        }
        // ------------------------------------------------------------------------------获取随机文件流结束

        // ------------------------------------------------------------------------
        // 写文件流到指定的文件开始
        try {
            // 创建文件随机访问对象
            randomAccessFile = new RandomAccessFile(saveFilePath, "rw");
            // 将文件写入位置移动到其实点
            randomAccessFile.seek(startIndex);
            // 写入文件
            int bytes = 0;
            byte[] buffer = new byte[100 * 1024];
            while ((bytes = fileInputStream.read(buffer, 0, buffer.length)) != -1) {
                randomAccessFile.write(buffer, 0, bytes);
            }
        } catch (IOException e) {
            e.printStackTrace();
        } finally {
            try {
                if (null != randomAccessFile) {
                    randomAccessFile.close();
                }
                if (null != fileInputStream) {
                    fileInputStream.close();
                }
            } catch (IOException e) {
                e.printStackTrace();
            }
        }
        // ------------------------------------------------------------------------写文件流到指定的文件结束

        System.out.println("线程名称[" + Thread.currentThread().getName() + "]于时间["
                + new SimpleDateFormat("yyyy-MM-dd HH:mm:ss").format(new Date()) + "]下载完成！");
    }

    public String getDownloadFilePath() {
        return downloadFilePath;
    }

    public void setDownloadFilePath(String downloadFilePath) {
        this.downloadFilePath = downloadFilePath;
    }

    public String getSaveFilePath() {
        return saveFilePath;
    }

    public void setSaveFilePath(String saveFilePath) {
        this.saveFilePath = saveFilePath;
    }

    public int getStartIndex() {
        return startIndex;
    }

    public void setStartIndex(int startIndex) {
        this.startIndex = startIndex;
    }

    public int getEndIndex() {
        return endIndex;
    }

    public void setEndIndex(int endIndex) {
        this.endIndex = endIndex;
    }

}

测试：

package com.bochao.download;

public class TestDownload {

	public static void main(String[] args) {
		
	    String downloadFilePath = "http://localhost:81/mulitThreadDownload/file/PowerDesigner165_Evaluation.1428562995.exe";
		String saveFileDir= "f:\\Download\\";
		
		HttpMulitThreadDownload httpMulitThreadDownload = new HttpMulitThreadDownload(2, downloadFilePath, saveFileDir, "powerdesigner.exe");
		httpMulitThreadDownload.MulitThreadDownload();
	}

}

输出结果：

线程名称[pool-1-thread-1]于时间[2015-08-30 16:09:49]开始下载....
线程名称[pool-1-thread-2]于时间[2015-08-30 16:09:49]开始下载....
线程名称[pool-1-thread-2]于时间[2015-09-30 16:09:01]下载完成！
线程名称[pool-1-thread-1]于时间[2015-09-30 16:09:23]下载完成！

本文章为转载内容，我们尊重原作者对文章享有的著作权。如有内容错误或侵权问题，欢迎原作者联系我们进行内容更正或删除文章。