oss断点续传

themis-admin/src/main/java/com/qmth/themis/admin/api/test.java

@@ -1,132 +0,0 @@
-package com.qmth.themis.admin.api;
-
-import java.io.*;
-import java.util.Map;
-import java.util.Objects;
-import java.util.StringJoiner;
-import java.util.concurrent.ConcurrentHashMap;
-import java.util.concurrent.CountDownLatch;
-
-public class test {
-
-    /**
-     * 断点续传
-     *
-     * @param dataStr   目标文件地址
-     * @param targetStr 存放地址
-     */
-    public static void breakpointResume(String dataStr, String targetStr) {
-        File dataFile = new File(dataStr);
-        long length = dataFile.length();
-        int threadNum = 4;//指定线程数
-        //每个线程均分文件大小，且向上取整
-        long part = (long) Math.ceil(length / threadNum);
-        //线程减法计数器
-        CountDownLatch countDownLatch = new CountDownLatch(threadNum);
-        long start = System.currentTimeMillis();
-        //记录传输的日志文件
-        File logFile = new File(targetStr + ".log");
-        String[] splitData = null;//不是null就需要断点续传
-        BufferedReader reader = null;
-        try {
-            if (logFile.exists()) {
-                //存在日志文件，需要进行断点续传
-                reader = new BufferedReader(new FileReader(logFile));
-                String data = reader.readLine();
-                splitData = data.split(",");
-            } else {
-                //不存在日志文件，创建日志文件
-                logFile.createNewFile();
-            }
-            Map<Integer, Long> maps = new ConcurrentHashMap<>();
-            for (int i = 0; i < threadNum; i++) {
-                final int k = i;
-                System.out.println("线程正在执行任务：" + k);
-                String[] finalData = splitData;
-                new Thread(() -> {
-                    RandomAccessFile inFile = null;
-                    RandomAccessFile outFile = null;
-                    RandomAccessFile rafLog = null;
-                    try {
-                        inFile = new RandomAccessFile(dataFile, "r");//读
-                        outFile = new RandomAccessFile(targetStr, "rw");//写
-                        rafLog = new RandomAccessFile(logFile, "rw");//操作日志文件的流
-                        //确定每个线程读取文件的开始和结束的位置,有断点续传就从日志文件取出的位置开始读取
-                        inFile.seek(finalData == null ? k * part : Long.parseLong(finalData[k]));//设置每个线程读取的启始位置
-                        outFile.seek(finalData == null ? k * part : Long.parseLong(finalData[k]));//设置每个线程写入的启始位置
-                        byte[] bytes = new byte[1024 * 10];//每次读取字节大小
-                        int len = -1, allLen = 0;
-                        while (true) {
-                            len = inFile.read(bytes);//从磁盘读取到缓存
-                            if (len == -1) { //数据读完，结束
-                                break;
-                            }
-                            //如果不等于 -1，把每次读取的字节累加
-                            allLen = allLen + len;
-                            //将读取的字节数放入到map中
-                            maps.put(k, allLen + (finalData == null ? k * part : Long.parseLong(finalData[k])));//每个线程的绝对偏移量
-                            outFile.write(bytes, 0, len);//从缓存写入到磁盘
-                            //将map中的字节日志信息数据写入磁盘
-                            StringJoiner stringJoiner = new StringJoiner(",");
-                            maps.forEach((key, value) -> stringJoiner.add(String.valueOf(value)));
-                            //将日志信息写入磁盘
-                            rafLog.seek(0);//覆盖之前的日志信息
-                            rafLog.write(stringJoiner.toString().getBytes("UTF-8"));
-                            /**
-                             * 当前线程读取的内容
-                             *  allLen + (k * part)
-                             *  或
-                             *  allLen + finalData[k] 日志文件里面的偏移量
-                             *  >=
-                             *  下个线程的起始部分((k + 1) * part)
-                             *  当前线程就不再读取写入数据，结束任务
-                             */
-                            if (allLen + (finalData == null ? k * part : Long.parseLong(finalData[k])) >= (k + 1) * part) {
-                                break;
-                            }
-                        }
-                    } catch (IOException e) {
-                        e.printStackTrace();
-                    } finally {
-                        //关流
-                        try {
-                            if (Objects.nonNull(outFile)) {
-                                outFile.close();
-                            }
-                            if (Objects.nonNull(inFile)) {
-                                inFile.close();
-                            }
-                            if (Objects.nonNull(rafLog)) {
-                                rafLog.close();
-                            }
-                        } catch (IOException e) {
-                            e.printStackTrace();
-                        }
-                        countDownLatch.countDown();//减一
-                    }
-                }).start();
-            }
-
-            //主线程要等到线程计数器归零，再继续往下执行
-            countDownLatch.await();
-            long end = System.currentTimeMillis();
-            System.out.println("总耗时：" + (end - start) / 1000 + "秒");
-            //删除日志文件
-            logFile.delete();
-        } catch (Exception e) {
-            e.printStackTrace();
-        } finally {
-            if (null != reader) {
-                try {
-                    reader.close();
-                } catch (IOException e) {
-                    e.printStackTrace();
-                }
-            }
-        }
-    }
-
-    public static void main(String[] args) {
-        breakpointResume("/Users/king/Downloads/cn_windows_10_multiple_editions_version_1511_x64_dvd_7223622.iso", "/Users/king/Downloads/new.iso");
-    }
-}

themis-business/src/main/java/com/qmth/themis/business/util/OssUtil.java

@@ -5,6 +5,7 @@ import com.aliyun.oss.OSS;
 import com.aliyun.oss.OSSClientBuilder;
 import com.aliyun.oss.common.utils.BinaryUtil;
 import com.aliyun.oss.model.*;
+import com.qmth.themis.business.constant.SystemConstant;
 import com.qmth.themis.business.domain.AliYunOssDomain;
 import com.qmth.themis.business.domain.AliYunOssPrivateDomain;
 import com.qmth.themis.business.domain.AliYunOssPublicDomain;
@@ -123,6 +124,7 @@ public class OssUtil {
      */
     public void upload(boolean isPublic, String objectName, File file) throws IOException {
         log.info("oss  upload file is come in");
+        long start = System.currentTimeMillis();
         String bucket = isPublic ? aliYunOssPublicDomain.getPublicBucket() : aliYunOssPrivateDomain.getPrivateBucket();
         OSS client = isPublic ? publicClient : privateClient;
         ObjectMetadata meta = new ObjectMetadata();
@@ -133,6 +135,8 @@ public class OssUtil {
         PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, objectName, file, meta);
         PutObjectResult por = client.putObject(putObjectRequest);
         log.info("objectName:{},requestid:{}", objectName, por.getRequestId());
+        long end = System.currentTimeMillis();
+        System.out.println("总耗时：" + (end - start) / 1000 + "秒");
     }
 
     /**
@@ -143,17 +147,46 @@ public class OssUtil {
      * @param file
      */
     public void uploadPart(boolean isPublic, String objectName, File file) throws IOException {
-        log.info("oss  upload file is come in");
+        log.info("oss uploadPart file is come in");
+        long start = System.currentTimeMillis();
         String bucket = isPublic ? aliYunOssPublicDomain.getPublicBucket() : aliYunOssPrivateDomain.getPrivateBucket();
         OSS client = isPublic ? publicClient : privateClient;
-        ObjectMetadata meta = new ObjectMetadata();
-        // 设置MD5校验。
-        String md5 = BinaryUtil.toBase64String(BinaryUtil.calculateMd5(FileUtils.readFileToByteArray(file)));
-        meta.setContentMD5(md5);
-        // 上传内容到指定的存储空间（bucketName）并保存为指定的文件名称（objectName）。
-        PutObjectRequest putObjectRequest = new PutObjectRequest(bucket, objectName, file, meta);
-        PutObjectResult por = client.putObject(putObjectRequest);
-        log.info("objectName:{},requestid:{}", objectName, por.getRequestId());
+        try {
+            ObjectMetadata meta = new ObjectMetadata();
+//             指定上传的内容类型。
+//            meta.setContentType("text/plain");
+            // 文件上传时设置访问权限ACL。
+            // meta.setObjectAcl(CannedAccessControlList.Private);
+            // 通过UploadFileRequest设置多个参数。
+            // 依次填写Bucket名称（例如examplebucket）以及Object完整路径（例如exampledir/exampleobject.txt），Object完整路径中不能包含Bucket名称。
+            UploadFileRequest uploadFileRequest = new UploadFileRequest(bucket, objectName);
+//                    .withProgressListener(new test());
+
+            // 通过UploadFileRequest设置单个参数。
+            // 填写本地文件的完整路径，例如D:\\localpath\\examplefile.txt。如果未指定本地路径，则默认从示例程序所属项目对应本地路径中上传文件。
+            uploadFileRequest.setUploadFile(file.getPath());
+            // 指定上传并发线程数，默认值为1。
+            uploadFileRequest.setTaskNum(5);
+            // 指定上传的分片大小，单位为字节，取值范围为100 KB~5 GB。默认值为100 KB。
+            uploadFileRequest.setPartSize(1 * 1024 * 1024);
+            // 开启断点续传，默认关闭。
+            uploadFileRequest.setEnableCheckpoint(true);
+            // 记录本地分片上传结果的文件。上传过程中的进度信息会保存在该文件中，如果某一分片上传失败，再次上传时会根据文件中记录的点继续上传。上传完成后，该文件会被删除。
+            // 如果未设置该值，默认与待上传的本地文件同路径，名称为${uploadFile}.ucp。
+//            uploadFileRequest.setCheckpointFile(SystemConstant.TEMP_FILES_DIR);
+            // 文件的元数据。
+            uploadFileRequest.setObjectMetadata(meta);
+            // 设置上传回调，参数为Callback类型。
+//            uploadFileRequest.setCallback(t.callback());
+            // 断点续传上传。
+            client.uploadFile(uploadFileRequest);
+            long end = System.currentTimeMillis();
+            System.out.println("总耗时：" + (end - start) / 1000 + "秒");
+        } catch (Exception e) {
+            log.error(SystemConstant.LOG_ERROR, e);
+        } catch (Throwable ce) {
+            log.error(SystemConstant.LOG_ERROR, ce);
+        }
     }
 
     /**
@@ -256,5 +289,4 @@ public class OssUtil {
         URL url = privateUrlClient.generatePresignedUrl(generatePresignedUrlRequest);
         return url.toString();
     }
-
 }