[chg]多线程压缩

朱海
Commit fa3dd3b2 authored Jun 20, 2022 by 朱海
Showing 1 changed file with 39 additions and 5 deletions
label-platform/src/main/java/com/viontech/label/platform/service/main/ReidService.java
--- a/label-platform/src/main/java/com/viontech/label/platform/service/main/ReidService.java
+++ b/label-platform/src/main/java/com/viontech/label/platform/service/main/ReidService.java
@@ -4,6 +4,7 @@ import cn.dev33.satoken.stp.StpUtil;
 import com.fasterxml.jackson.core.JsonProcessingException;
 import com.fasterxml.jackson.databind.ObjectMapper;
 import com.google.common.collect.Lists;
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
 import com.viontech.keliu.model.AlgResult;
 import com.viontech.keliu.model.BodyFeature;
 import com.viontech.keliu.model.Data;
@@ -31,6 +32,11 @@ import com.viontech.label.platform.vo.PicVo;
 import com.viontech.label.platform.vo.ReidUploadData;
 import com.viontech.label.platform.websocket.ReidWebsocket;
 import lombok.extern.slf4j.Slf4j;
+import org.apache.commons.compress.archivers.zip.ParallelScatterZipCreator;
+import org.apache.commons.compress.archivers.zip.UnixStat;
+import org.apache.commons.compress.archivers.zip.ZipArchiveEntry;
+import org.apache.commons.compress.archivers.zip.ZipArchiveOutputStream;
+import org.apache.commons.compress.parallel.InputStreamSupplier;
 import org.apache.commons.lang3.ArrayUtils;
 import org.redisson.api.RMap;
 import org.redisson.api.RSet;
@@ -39,9 +45,11 @@ import org.springframework.beans.factory.annotation.Autowired;
 import org.springframework.data.redis.core.RedisTemplate;
 import org.springframework.stereotype.Service;
 import org.springframework.transaction.annotation.Transactional;
+import org.springframework.util.StopWatch;
 import org.springframework.web.multipart.MultipartFile;
 import javax.annotation.Resource;
+import java.io.ByteArrayInputStream;
 import java.io.File;
 import java.io.FileOutputStream;
 import java.util.ArrayList;
@@ -57,8 +65,12 @@ import java.util.Objects;
 import java.util.Set;
 import java.util.UUID;
 import java.util.concurrent.CompletableFuture;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
 import java.util.concurrent.TimeUnit;
 import java.util.stream.Collectors;
+import java.util.zip.Deflater;
 import java.util.zip.ZipEntry;
 import java.util.zip.ZipOutputStream;
@@ -892,17 +904,36 @@ public class ReidService {
    public File downloadLabeledPicAsZip(Long packId, Integer status) throws Exception {
        File tempFile = File.createTempFile("label-" + packId, ".zip");
-        try (ZipOutputStream zipOutputStream = new ZipOutputStream(new FileOutputStream(tempFile))) {
+        ExecutorService executor = new ThreadPoolExecutor(6, 20, 60, TimeUnit.SECONDS, new LinkedBlockingQueue<>(20),
+                new ThreadFactoryBuilder().setNameFormat("compressFileList-pool-").build(), new ThreadPoolExecutor.CallerRunsPolicy());
+        ParallelScatterZipCreator parallelScatterZipCreator = new ParallelScatterZipCreator(executor);
+        try (ZipArchiveOutputStream zipOutputStream = new ZipArchiveOutputStream (new FileOutputStream(tempFile))) {
+            zipOutputStream.setLevel(Deflater.BEST_SPEED);
            SubTaskExample subTaskExample = new SubTaskExample();
            subTaskExample.createCriteria().andPackIdEqualTo(packId).andStatusEqualTo(status);
            List<SubTask> allSubTask = subTaskService.selectByExample(subTaskExample);
+            StopWatch stopWatch = new StopWatch();
+            stopWatch.start("压缩文件");
+            for (SubTask subTask : allSubTask) {
+                PicVo pic = storageUtils.getPicWithImg(subTask.getPicId());
+                if (pic == null) {
+                    continue;
+                }
+                byte[] image = pic.getImage();
+                if (image != null) {
+                    InputStreamSupplier inputStreamSupplier = () -> new ByteArrayInputStream(image);
+                    ZipArchiveEntry zipArchiveEntry = new ZipArchiveEntry(subTask.getPersonUnid() + "/" + pic.getName());
+                    zipArchiveEntry.setMethod(ZipArchiveEntry.DEFLATED);
+                    zipArchiveEntry.setUnixMode(UnixStat.FILE_FLAG | 436);
+                    parallelScatterZipCreator.addArchiveEntry(zipArchiveEntry, inputStreamSupplier);
+                }
+            }
+            parallelScatterZipCreator.writeTo(zipOutputStream);
+            /*Map<String, List<SubTask>> collect = allSubTask.stream().collect(Collectors.groupingBy(SubTask::getPersonUnid, Collectors.toList()));
-            Map<String, List<SubTask>> collect = allSubTask.stream().collect(Collectors.groupingBy(SubTask::getPersonUnid, Collectors.toList()));
            for (Map.Entry<String, List<SubTask>> entry : collect.entrySet()) {
                String personUnid = entry.getKey();
                List<SubTask> subtasks = entry.getValue();
                for (SubTask subTask : subtasks) {
                    PicVo pic = storageUtils.getPicWithImg(subTask.getPicId());
                    if (pic == null) {
@@ -915,7 +946,10 @@ public class ReidService {
                        zipOutputStream.write(image);
                    }
                }
-            }
+            }*/
+            stopWatch.stop();
+            log.info("压缩文件用时：{}", stopWatch.prettyPrint());
        }
        return tempFile;
    }