diff --git a/src/main/java/com/ilummc/eagletdl/AlreadyStartException.java b/src/main/java/com/ilummc/eagletdl/AlreadyStartException.java new file mode 100644 index 0000000..13919a0 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/AlreadyStartException.java @@ -0,0 +1,4 @@ +package com.ilummc.eagletdl; + +public class AlreadyStartException extends RuntimeException { +} diff --git a/src/main/java/com/ilummc/eagletdl/CompleteEvent.java b/src/main/java/com/ilummc/eagletdl/CompleteEvent.java new file mode 100644 index 0000000..839625b --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/CompleteEvent.java @@ -0,0 +1,19 @@ +package com.ilummc.eagletdl; + +public class CompleteEvent { + private EagletTask task; + private boolean success; + + CompleteEvent(EagletTask task, boolean success) { + this.task = task; + this.success = success; + } + + public boolean isSuccess() { + return success; + } + + public EagletTask getTask() { + return task; + } +} diff --git a/src/main/java/com/ilummc/eagletdl/ConnectedEvent.java b/src/main/java/com/ilummc/eagletdl/ConnectedEvent.java new file mode 100644 index 0000000..5299f27 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/ConnectedEvent.java @@ -0,0 +1,27 @@ +package com.ilummc.eagletdl; + +public class ConnectedEvent { + + private long contentLength; + private EagletTask task; + + public ConnectedEvent(long length, EagletTask task) { + this.contentLength = length; + this.task = task; + } + + /** + * Get the length of the download task. + *

+ * If the length is -1, this task cannot be downloaded in multiple threads. + * + * @return length + */ + public long getContentLength() { + return contentLength; + } + + public EagletTask getTask() { + return task; + } +} diff --git a/src/main/java/com/ilummc/eagletdl/DoNotSupportMultipleThreadException.java b/src/main/java/com/ilummc/eagletdl/DoNotSupportMultipleThreadException.java new file mode 100644 index 0000000..0c59692 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/DoNotSupportMultipleThreadException.java @@ -0,0 +1,4 @@ +package com.ilummc.eagletdl; + +public class DoNotSupportMultipleThreadException extends RuntimeException { +} diff --git a/src/main/java/com/ilummc/eagletdl/Eaglet.java b/src/main/java/com/ilummc/eagletdl/Eaglet.java new file mode 100644 index 0000000..7e000b3 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/Eaglet.java @@ -0,0 +1,22 @@ +package com.ilummc.eagletdl; + +/** + * Test class + */ +public class Eaglet { + + public static void main(String[] args) { + //new EagletTask().url("http://sgp-ping.vultr.com/vultr.com.100MB.bin") + new EagletTask().url("https://gitee.com/bkm016/TabooLibCloud/raw/master/TabooMenu/TabooMenu.jar") + .file("F:\\test.dl") + .setThreads(1) + .readTimeout(1000) + .connectionTimeout(1000) + .maxRetry(30) + .setOnConnected(event -> System.out.println(event.getContentLength())) + .setOnProgress(event -> System.out.println(event.getSpeedFormatted() + " " + event.getPercentageFormatted())) + .setOnComplete(event -> System.out.println("Complete")) + .start(); + } + +} diff --git a/src/main/java/com/ilummc/eagletdl/EagletHandler.java b/src/main/java/com/ilummc/eagletdl/EagletHandler.java new file mode 100644 index 0000000..ef63e4a --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/EagletHandler.java @@ -0,0 +1,8 @@ +package com.ilummc.eagletdl; + +@FunctionalInterface +public interface EagletHandler { + + void handle(T event) ; + +} diff --git a/src/main/java/com/ilummc/eagletdl/EagletTask.java b/src/main/java/com/ilummc/eagletdl/EagletTask.java new file mode 100644 index 0000000..fb85bdb --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/EagletTask.java @@ -0,0 +1,460 @@ +package com.ilummc.eagletdl; + +import java.io.File; +import java.net.HttpURLConnection; +import java.net.MalformedURLException; +import java.net.Proxy; +import java.net.URL; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.concurrent.ConcurrentHashMap; +import java.util.concurrent.ExecutorService; +import java.util.concurrent.Executors; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.locks.ReentrantLock; + +public class EagletTask { + + private ReentrantLock lock = new ReentrantLock(); + + Map httpHeader = new ConcurrentHashMap<>(); + + private URL url; + + EagletHandler onError = event -> event.getException().printStackTrace(); + + private EagletHandler onStart; + + private EagletHandler onComplete; + + private EagletHandler onConnected; + + private EagletHandler onProgress; + + private Proxy proxy; + + private String md5, sha1, sha256; + + String requestMethod = "GET"; + + private int threadAmount = 1; + + int connectionTimeout = 7000; + int readTimeout = 7000; + int maxRetry = 3; + + private File dest; + + private transient boolean running = false; + private transient long contentLength, maxBlockingTime = 7000; + private transient ExecutorService executorService; + private transient Thread monitor; + + public EagletTask() { + } + + /** + * Stop this task forcefully, and the target file will not be removed. + */ + public void stop() { + executorService.shutdownNow(); + monitor.interrupt(); + } + + /** + * Start the download file + *

+ * 开始下载文件 + */ + public EagletTask start() { + // create thread pool for download + executorService = Executors.newFixedThreadPool(threadAmount); + // check if is already running + if (running) throw new AlreadyStartException(); + // start the monitor thread + monitor = new Thread(() -> { + lock.lock(); + // fire a new start event + if (onStart != null) onStart.handle(new StartEvent(this)); + try { + // create the target file + if (!dest.exists()) dest.createNewFile(); + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + // set the connection properties + httpHeader.forEach(connection::addRequestProperty); + connection.setRequestMethod(requestMethod); + connection.setConnectTimeout(30000); + connection.setReadTimeout(30000); + connection.connect(); + contentLength = connection.getContentLengthLong(); + // fire a new connected event + // contains connection properties + if (onConnected != null) onConnected.handle(new ConnectedEvent(contentLength, this)); + // if this is an unknown length task + if (contentLength == -1 || threadAmount == 1) { + // pass the connection instance to this new thread + SingleThreadDownload download = new SingleThreadDownload(connection, dest, this); + executorService.execute(download); + long last = 0; + while (true) { + Thread.sleep(1000); + // check the progress + long progress = download.getCurrentProgress(); + // fire a new progress event + if (onProgress != null) + onProgress.handle(new ProgressEvent(progress - last, this, + ((double) progress) / Math.max((double) contentLength, 0D))); + last = progress; + // check complete + if (last == contentLength || download.isComplete()) { + break; + } + } + // close the thread pool, release resources + executorService.shutdown(); + // change the running flag to false + running = false; + } else { + List splitDownloads = new ArrayList<>(); + // Assign download task length + long blockSize = contentLength / threadAmount; + for (int threadId = 0; threadId < threadAmount; threadId++) { + long startIndex = threadId * blockSize; + long endIndex = (threadId + 1) * blockSize - 1; + if (threadId == (threadAmount - 1)) { + endIndex = contentLength - 1; + } + SplitDownload download = new SplitDownload(url, startIndex, endIndex, dest, this); + // Start downloading + executorService.execute(download); + splitDownloads.add(download); + } + long last = 0; + while (true) { + Thread.sleep(1000); + long progress = 0; + // Collect download progress + for (SplitDownload splitDownload : splitDownloads) { + progress += splitDownload.getCurrentIndex() - splitDownload.startIndex; + // blocked then restart from current index + if (!splitDownload.isComplete() && + System.currentTimeMillis() - splitDownload.getLastUpdateTime() > maxBlockingTime) { + splitDownload.setStartIndex(splitDownload.getCurrentIndex()); + if (splitDownload.getRetry() <= maxRetry) + executorService.execute(splitDownload); + else throw new RetryFailedException(this); + } + } + // Fire a progress event + if (onProgress != null) + onProgress.handle(new ProgressEvent(progress - last, this, + ((double) progress) / ((double) contentLength))); + last = progress; + // check complete + if (last >= contentLength) { + break; + } + } + // close the thread pool, release resources + executorService.shutdown(); + // change the running flag to false + running = false; + } + // check hash + if (md5 != null && !md5.equalsIgnoreCase(HashUtil.md5(dest))) + throw new HashNotMatchException(); + if (sha1 != null && !sha1.equalsIgnoreCase(HashUtil.sha1(dest))) + throw new HashNotMatchException(); + if (sha256 != null && !sha256.equalsIgnoreCase(HashUtil.sha256(dest))) + throw new HashNotMatchException(); + if (onComplete != null) onComplete.handle(new CompleteEvent(this, true)); + } catch (Exception e) { + onError.handle(new ErrorEvent(e, this)); + executorService.shutdown(); + if (onComplete != null) onComplete.handle(new CompleteEvent(this, false)); + } finally { + lock.unlock(); + } + }, "EagletTaskMonitor"); + monitor.start(); + return this; + } + + public EagletTask waitUntil() { + while (lock.tryLock()) lock.unlock(); + lock.lock(); + lock.unlock(); + return this; + } + + public EagletTask waitFor(long timeout, TimeUnit unit) { + while (lock.tryLock()) lock.unlock(); + try { + lock.tryLock(timeout, unit); + } catch (InterruptedException e) { + e.printStackTrace(); + } + return this; + } + + public EagletTask maxRetry(int maxRetry) { + this.maxRetry = maxRetry; + return this; + } + + /** + * Set the sha256 hash of the download file. Case is not sensitive. + *

+ * If the hash check failed, an error event will be fired. + * + * @param sha256 file sha1 + * @return task instance + */ + public EagletTask sha256(String sha256) { + this.sha256 = sha256; + return this; + } + + /** + * Set the sha1 hash of the download file. Case is not sensitive. + *

+ * If the hash check failed, an error event will be fired. + * + * @param sha1 file sha1 + * @return task instance + */ + public EagletTask sha1(String sha1) { + this.sha1 = sha1; + return this; + } + + /** + * Set the md5 hash of the download file. Case is not sensitive. + *

+ * If the hash check failed, an error event will be fired. + * + * @param md5 file md5 + * @return task instance + */ + public EagletTask md5(String md5) { + this.md5 = md5; + return this; + } + + /** + * Set the max blocked time per download thread. + *

+ * If the thread blocks exceeded the provided time, this thread will re-start the task. + * + * @param maxBlockingTime time + * @return task instance + */ + private EagletTask maxBlocking(long maxBlockingTime) { + this.maxBlockingTime = maxBlockingTime; + return this; + } + + /** + * Set the progress handler + *

+ * This handler will be called every 1000 milli seconds. + *

+ * 设置处理进度的时间监听器。该监听器的 handle 方法每秒调用一次。 + * + * @param onProgress handler + * @return task instance + */ + public EagletTask setOnProgress(EagletHandler onProgress) { + this.onProgress = onProgress; + return this; + } + + /** + * Set the download file + * + * @param file the file's absolute path + * @return task instance + */ + public EagletTask file(String file) { + this.dest = new File(file); + return this; + } + + /** + * Set the download file + * + * @param file the file + * @return task instance + */ + public EagletTask file(File file) { + this.dest = file; + return this; + } + + /** + * Set the connected handler + *

+ * This will be called when the connection is established + *

+ * Async call + * + * @param onConnected onConnected event handler + * @return task instance + */ + public EagletTask setOnConnected(EagletHandler onConnected) { + this.onConnected = onConnected; + return this; + } + + /** + * Set the read timeout, default is 7000 + * + * @param timeout timeout + * @return task instance + */ + public EagletTask readTimeout(int timeout) { + this.readTimeout = timeout; + return this; + } + + /** + * Set the connection timeout, default is 7000 + * + * @param timeout timeout + * @return task instance + */ + public EagletTask connectionTimeout(int timeout) { + this.connectionTimeout = timeout; + return this; + } + + /** + * Set the request method, default is GET + * + * @param requestMethod the request method + * @return task instance + */ + public EagletTask requestMethod(String requestMethod) { + this.requestMethod = requestMethod; + return this; + } + + /** + * Set the complete event handler + *

+ * This handler will be called when everything is complete, and the downloaded file is available + *

+ * Async call + * + * @param onComplete the handler + * @return task instance + */ + public EagletTask setOnComplete(EagletHandler onComplete) { + this.onComplete = onComplete; + return this; + } + + /** + * Set the start handler + *

+ * This handler will be called when the start method is called + *

+ * Async call + * + * @param onStart the handler + * @return task instance + */ + public EagletTask setOnStart(EagletHandler onStart) { + this.onStart = onStart; + return this; + } + + /** + * Set the network proxy + * + * @param proxy the proxy + * @return task instance + */ + public EagletTask proxy(Proxy proxy) { + this.proxy = proxy; + return this; + } + + /** + * Set the error handler, default is to print the stack trace + *

+ * This handler will be called when an exception is thrown + *

+ * Async call + * + * @param onError the handler + * @return task instance + */ + public EagletTask setOnError(EagletHandler onError) { + this.onError = onError; + return this; + } + + + /** + * Set how much thread should be used to download, default is 1 + * + * @param i thread amount + * @return task instance + */ + public EagletTask setThreads(int i) { + if (i < 1) throw new RuntimeException("Thread amount cannot be zero or negative!"); + threadAmount = i; + return this; + } + + /** + * Set the download source + * + * @param url the url + * @return task instance + */ + public EagletTask url(URL url) { + this.url = url; + return this; + } + + /** + * Set the download source + * + * @param url the url + * @return task instance + */ + public EagletTask url(String url) { + try { + this.url = new URL(url); + } catch (MalformedURLException e) { + onError.handle(new ErrorEvent(e, this)); + } + return this; + } + + /** + * Clear the http header field + * + * @return task instance + */ + public EagletTask clearHeaders() { + httpHeader.clear(); + return this; + } + + /** + * Set the header field of the http request + * + * @param key header key + * @param value header value + * @return builder instance + */ + public EagletTask header(String key, String value) { + httpHeader.put(key, value); + return this; + } + +} diff --git a/src/main/java/com/ilummc/eagletdl/ErrorEvent.java b/src/main/java/com/ilummc/eagletdl/ErrorEvent.java new file mode 100644 index 0000000..801ac9d --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/ErrorEvent.java @@ -0,0 +1,20 @@ +package com.ilummc.eagletdl; + +public class ErrorEvent { + + private Throwable e; + private EagletTask task; + + public ErrorEvent(Throwable e, EagletTask task) { + this.e = e; + this.task = task; + } + + public EagletTask getTask() { + return task; + } + + public Throwable getException() { + return e; + } +} diff --git a/src/main/java/com/ilummc/eagletdl/HashNotMatchException.java b/src/main/java/com/ilummc/eagletdl/HashNotMatchException.java new file mode 100644 index 0000000..bdb3bcd --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/HashNotMatchException.java @@ -0,0 +1,4 @@ +package com.ilummc.eagletdl; + +public class HashNotMatchException extends RuntimeException { +} diff --git a/src/main/java/com/ilummc/eagletdl/HashUtil.java b/src/main/java/com/ilummc/eagletdl/HashUtil.java new file mode 100644 index 0000000..10bf013 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/HashUtil.java @@ -0,0 +1,63 @@ +package com.ilummc.eagletdl; + +import java.io.File; +import java.io.FileInputStream; +import java.io.IOException; +import java.math.BigInteger; +import java.security.MessageDigest; +import java.security.NoSuchAlgorithmException; + +public class HashUtil { + + public static String sha256(File file) { + try { + FileInputStream fis = new FileInputStream(file); + MessageDigest md = MessageDigest.getInstance("SHA256"); + byte[] buffer = new byte[1024]; + int length = -1; + while ((length = fis.read(buffer, 0, 1024)) != -1) { + md.update(buffer, 0, length); + } + BigInteger bigInt = new BigInteger(1, md.digest()); + return bigInt.toString(16); + } catch (NoSuchAlgorithmException | IOException e) { + e.printStackTrace(); + } + return null; + } + + public static String sha1(File file) { + try { + FileInputStream fis = new FileInputStream(file); + MessageDigest md = MessageDigest.getInstance("SHA1"); + byte[] buffer = new byte[1024]; + int length = -1; + while ((length = fis.read(buffer, 0, 1024)) != -1) { + md.update(buffer, 0, length); + } + BigInteger bigInt = new BigInteger(1, md.digest()); + return bigInt.toString(16); + } catch (NoSuchAlgorithmException | IOException e) { + e.printStackTrace(); + } + return null; + } + + public static String md5(File file) { + try { + FileInputStream fis = new FileInputStream(file); + MessageDigest md = MessageDigest.getInstance("MD5"); + byte[] buffer = new byte[1024]; + int length = -1; + while ((length = fis.read(buffer, 0, 1024)) != -1) { + md.update(buffer, 0, length); + } + BigInteger bigInt = new BigInteger(1, md.digest()); + return bigInt.toString(16); + } catch (NoSuchAlgorithmException | IOException e) { + e.printStackTrace(); + } + return null; + } + +} diff --git a/src/main/java/com/ilummc/eagletdl/ProgressEvent.java b/src/main/java/com/ilummc/eagletdl/ProgressEvent.java new file mode 100644 index 0000000..f6cf1b7 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/ProgressEvent.java @@ -0,0 +1,53 @@ +package com.ilummc.eagletdl; + +import java.text.DecimalFormat; + +public class ProgressEvent { + + private long speed; + private EagletTask task; + private double percentage; + + ProgressEvent(long speed, EagletTask task, double percentage) { + this.speed = speed; + this.task = task; + this.percentage = percentage; + } + + public EagletTask getTask() { + return task; + } + + public long getSpeed() { + return speed; + } + + public double getPercentage() { + return percentage; + } + + public String getPercentageFormatted() { + return formatDouble(percentage * 100D) + " %"; + } + + /** + * Get the speed with format like X.00 MiB, Y.50 GiB, etc. + * + * @return formatted speed string + */ + public String getSpeedFormatted() { + return format(getSpeed()); + } + + private static String formatDouble(double d) { + return new DecimalFormat("0.00").format(d); + } + + public static String format(long l) { + if (l < 1024) return l + " B"; + if (l < 1024 * 1024) return formatDouble((double) l / 1024D) + " KiB"; + if (l < 1024 * 1024 * 1024) return formatDouble((double) l / (1024D * 1024D)) + " MiB"; + if (l < 1024 * 1024 * 1024 * 1024L) return formatDouble((double) l / (1024D * 1024D * 1024)) + " GiB"; + return ""; + } +} diff --git a/src/main/java/com/ilummc/eagletdl/RetryFailedException.java b/src/main/java/com/ilummc/eagletdl/RetryFailedException.java new file mode 100644 index 0000000..e8a022d --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/RetryFailedException.java @@ -0,0 +1,14 @@ +package com.ilummc.eagletdl; + +public class RetryFailedException extends RuntimeException { + + private EagletTask task; + + RetryFailedException(EagletTask task) { + this.task = task; + } + + public EagletTask getTask() { + return task; + } +} diff --git a/src/main/java/com/ilummc/eagletdl/SingleThreadDownload.java b/src/main/java/com/ilummc/eagletdl/SingleThreadDownload.java new file mode 100644 index 0000000..137762d --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/SingleThreadDownload.java @@ -0,0 +1,50 @@ +package com.ilummc.eagletdl; + +import java.io.*; +import java.net.HttpURLConnection; + +class SingleThreadDownload implements Runnable { + + private HttpURLConnection connection; + private File target; + private EagletTask task; + + private transient long currentProgress = 0, lastUpdateTime = System.currentTimeMillis(); + + private transient boolean complete = false; + + SingleThreadDownload(HttpURLConnection connection, File target, EagletTask task) { + this.connection = connection; + this.target = target; + this.task = task; + } + + long getLastUpdateTime() { + return lastUpdateTime; + } + + long getCurrentProgress() { + return currentProgress; + } + + public boolean isComplete() { + return complete; + } + + @Override + public void run() { + byte[] buf = new byte[1024]; + int len = 0; + try (BufferedInputStream stream = new BufferedInputStream(connection.getInputStream()); + BufferedOutputStream outputStream = new BufferedOutputStream(new FileOutputStream(target))) { + while ((len = stream.read(buf)) > 0) { + outputStream.write(buf, 0, len); + currentProgress += len; + lastUpdateTime = System.currentTimeMillis(); + } + } catch (IOException e) { + task.onError.handle(new ErrorEvent(e, task)); + } + complete = true; + } +} diff --git a/src/main/java/com/ilummc/eagletdl/SplitDownload.java b/src/main/java/com/ilummc/eagletdl/SplitDownload.java new file mode 100644 index 0000000..6727537 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/SplitDownload.java @@ -0,0 +1,89 @@ +package com.ilummc.eagletdl; + +import java.io.BufferedInputStream; +import java.io.File; +import java.io.RandomAccessFile; +import java.net.HttpURLConnection; +import java.net.URL; + +class SplitDownload implements Runnable { + + private URL url; + long startIndex, endIndex; + private File target; + private EagletTask task; + + private transient long currentIndex, lastUpdateTime = System.currentTimeMillis(), tmpStart; + private transient int retry = 0; + private transient boolean complete; + + SplitDownload(URL url, long startIndex, long endIndex, File dest, EagletTask task) { + this.url = url; + tmpStart = this.startIndex = this.currentIndex = startIndex; + this.endIndex = endIndex; + target = dest; + this.task = task; + } + + void setStartIndex(long index) { + this.tmpStart = index; + } + + long getLastUpdateTime() { + return lastUpdateTime; + } + + long getCurrentIndex() { + return currentIndex; + } + + int getRetry() { + return retry; + } + + boolean isComplete() { + return complete || currentIndex == endIndex + 1; + } + + @Override + public void run() { + try { + complete = false; + currentIndex = tmpStart; + HttpURLConnection connection = (HttpURLConnection) url.openConnection(); + // set the connection properties + task.httpHeader.forEach(connection::addRequestProperty); + connection.setRequestMethod(task.requestMethod); + connection.setConnectTimeout(task.connectionTimeout); + connection.setReadTimeout(task.readTimeout); + // set the download range + connection.setRequestProperty("Range", "bytes=" + tmpStart + "-" + endIndex); + connection.connect(); + // if response code not equals 206, it means that the server do not support multi thread downloading + if (connection.getResponseCode() == 206) { + RandomAccessFile file = new RandomAccessFile(target, "rwd"); + file.seek(tmpStart); + byte[] buf = new byte[1024]; + int len; + try (BufferedInputStream stream = new BufferedInputStream(connection.getInputStream())) { + while ((len = stream.read(buf)) > 0) { + file.write(buf, 0, len); + lastUpdateTime = System.currentTimeMillis(); + currentIndex += len; + // some mysterious error occurred while downloading + if (currentIndex >= endIndex + 2) { + currentIndex = tmpStart; + lastUpdateTime = 0; + retry++; + return; + } + } + complete = true; + } + } else throw new DoNotSupportMultipleThreadException(); + } catch (Exception e) { + task.onError.handle(new ErrorEvent(e, task)); + retry++; + } + } +} diff --git a/src/main/java/com/ilummc/eagletdl/StartEvent.java b/src/main/java/com/ilummc/eagletdl/StartEvent.java new file mode 100644 index 0000000..b0c7f61 --- /dev/null +++ b/src/main/java/com/ilummc/eagletdl/StartEvent.java @@ -0,0 +1,14 @@ +package com.ilummc.eagletdl; + +public class StartEvent { + + private EagletTask task; + + StartEvent(EagletTask task) { + this.task = task; + } + + public EagletTask getTask() { + return task; + } +}