优化图片压缩下载使用多线程

rain

2024-07-20 bf7a9ef439e3b49268974d564107999619b2b4fd

优化图片压缩下载使用多线程

 src/main/java/com/dji/sample/media/controller/FileController.java

@@ -171,23 +171,9 @@
        }
    }


    private static final String BASE_URL = "http://dev.jxpskj.com:9000/cloud-bucket";

    @GetMapping("/download-images")
    public void downloadImages(@RequestParam String jobId, HttpServletResponse response) {
        List<MediaFileEntity> entities = fileService.getMedia(jobId);
        List<String> urls = entities.stream()
                .map(MediaFileEntity::getObjectKey)
                .map(objectKey -> BASE_URL + objectKey)
                .collect(Collectors.toList());

        String outputFolder = "images"; // 存放图片的文件夹
        new File(outputFolder).mkdirs(); // 创建文件夹

        downloadAndSaveImages(urls, outputFolder);
        zipAndSendFolder(outputFolder, response);
        cleanUp(outputFolder);
    fileService.downloadImages(jobId,response);
    }

}

 src/main/java/com/dji/sample/media/dao/IFileMapper.java

@@ -8,6 +8,7 @@
import com.dji.sample.media.model.MediaJobDTO;
import com.dji.sample.media.model.param.SearchMediaParam;
import com.dji.sample.wayline.model.dto.WaylineJobDTO;
import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;

/**
@@ -15,6 +16,7 @@
 * @version 0.2
 * @date 2021/12/9
 */
@Mapper
public interface IFileMapper extends BaseMapper<MediaFileEntity> {
    Page<MediaFileEntity> getPage(@Param("page") Page<MediaFileEntity> mediaFileEntityPage, @Param("workspaceId") String workspaceId,@Param("query") MediaFileQueryParam mediaFileQueryParam);


 src/main/java/com/dji/sample/media/service/IFileService.java

@@ -5,7 +5,9 @@
import com.dji.sample.common.model.ResponseResult;
import com.dji.sample.media.model.*;
import com.drew.imaging.ImageProcessingException;
import org.springframework.web.bind.annotation.RequestParam;

import javax.servlet.http.HttpServletResponse;
import java.awt.*;
import java.io.IOException;
import java.net.URL;
@@ -102,6 +104,8 @@
     */
    URL getObjectUrl(String workspaceId, String fileId);

    void downloadImages( String jobId, HttpServletResponse response);

    /**
     * Query all media files of a job.
     *

 src/main/java/com/dji/sample/media/service/impl/FileServiceImpl.java

@@ -43,6 +43,7 @@
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;

import javax.servlet.http.HttpServletResponse;
import java.awt.*;
import java.io.File;
import java.io.FileInputStream;
@@ -54,6 +55,8 @@
import java.util.*;
import java.util.List;
import java.util.stream.Collectors;

import static com.dji.sample.media.util.ImageDownloaderAndCompressor.*;

/**
 * @author sean
@@ -429,6 +432,27 @@
    }

    @Override
    public void downloadImages(String jobId, HttpServletResponse response) {
        String address = pojo.getEndpoint() + "/" + pojo.getBucket();
        List<MediaFileEntity> entities = getMedia(jobId);
        List<String> urls = entities.stream()
                .map(MediaFileEntity::getObjectKey)
                .map(objectKey -> address + objectKey)
                .collect(Collectors.toList());
        String outputFolder = "images"; // 存放图片的文件夹
        new File(outputFolder).mkdirs(); // 创建文件夹
        try {
            downloadAndSaveImages(urls, outputFolder);
            zipAndSendFolder(outputFolder, response);
        } catch (Exception e) {
            e.printStackTrace();
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
        } finally {
            cleanUp(outputFolder);
        }
    }

    @Override
    public List<MediaFileDTO> getFilesByWorkspaceAndJobId(String workspaceId, String jobId) {
        return mapper.selectList(new LambdaQueryWrapper<MediaFileEntity>()
                        .eq(MediaFileEntity::getWorkspaceId, workspaceId)
@@ -791,43 +815,58 @@
    }
    @Override
    public PaginationData<MediaFileEntity> getJobId(int pageNum, int pageSize, String workspaceId) {
        // 分页查询
        Page<MediaFileEntity> page = new Page<>(pageNum, pageSize);
        Page<MediaFileEntity> resultPage = mapper.selectPage(page, new LambdaQueryWrapper<MediaFileEntity>()
                .eq(MediaFileEntity::getWorkspaceId, workspaceId));
        List<MediaFileEntity> allRecords = new ArrayList<>();
        int currentPageNum = 1;  // 从第一页开始获取记录

        List<MediaFileEntity> result = resultPage.getRecords();
        // 用于存储去重后的记录
        Map<String, MediaFileEntity> uniqueFilesMap = new HashMap<>();

        // 根据 fileName 字段进行去重并设置 JobName
        Map<String, MediaFileEntity> uniqueFilesMap = result.stream()
                .peek(mediaFile -> {
                    String taskNameResult = waylineJobService.getName(mediaFile.getJobId());
                    mediaFile.setJobName(taskNameResult);
                })
                .collect(Collectors.toMap(
                        MediaFileEntity::getFileName,
                        entity -> entity,
                        (existing, replacement) -> existing // 如果有重复的，保留已有的
                ));
        while (true) {
            // 分页查询
            Page<MediaFileEntity> page = new Page<>(currentPageNum, pageSize);
            Page<MediaFileEntity> resultPage = mapper.selectPage(page, new LambdaQueryWrapper<MediaFileEntity>()
                    .eq(MediaFileEntity::getWorkspaceId, workspaceId));

            List<MediaFileEntity> result = resultPage.getRecords();

            // 检查是否还有更多记录
            if (result.isEmpty()) {
                break;  // 没有更多记录了，停止请求
            }

            // 根据 fileName 字段进行去重并设置 JobName
            result.stream()
                    .peek(mediaFile -> {
                        String taskNameResult = waylineJobService.getName(mediaFile.getJobId());
                        mediaFile.setJobName(taskNameResult);
                    })
                    .forEach(mediaFile -> uniqueFilesMap.putIfAbsent(mediaFile.getFileName(), mediaFile));

            // 增加当前页数以获取更多记录
            currentPageNum++;
        }

        // 获取去重后的实际总数
        List<MediaFileEntity> uniqueFiles = new ArrayList<>(uniqueFilesMap.values());

        // 使用去重后的列表重新计算总数
        int uniqueTotal = uniqueFiles.size();

        // 计算当前页的起始和结束索引
        int fromIndex = Math.min((pageNum - 1) * pageSize, uniqueTotal);
        int toIndex = Math.min(fromIndex + pageSize, uniqueTotal);

        // 获取当前页的数据
        List<MediaFileEntity> pagedUniqueFiles = uniqueFiles.subList(fromIndex, toIndex);

        // 创建新的分页对象，使用去重后的总数
        Page<MediaFileEntity> uniquePage = new Page<>(pageNum, pageSize, uniqueTotal);

        // 截取当前页的数据
        int fromIndex = Math.min((pageNum - 1) * pageSize, uniqueTotal);
        int toIndex = Math.min(fromIndex + pageSize, uniqueTotal);
        List<MediaFileEntity> pagedUniqueFiles = uniqueFiles.subList(fromIndex, toIndex);

        // 使用新的分页对象构造 Pagination
        Pagination pagination = new Pagination(uniquePage);

        return new PaginationData<>(pagedUniqueFiles, pagination);
    }




    @Override
@@ -870,9 +909,12 @@



    public List<MediaFileEntity> getMedia(String jobId){
      return   mapper.selectList(new LambdaQueryWrapper<MediaFileEntity>().eq(MediaFileEntity::getJobId,jobId));
    public List<MediaFileEntity> getMedia(String jobId) {
        return mapper.selectList(new LambdaQueryWrapper<MediaFileEntity>()
                .eq(MediaFileEntity::getJobId, jobId)
                .groupBy(MediaFileEntity::getFileName));  // 使用groupBy去重
    }

    public void updateById(Integer id, MediaFileMarkEntity entity) {
        entity.setIsadd(1);
        UpdateWrapper<MediaFileMarkEntity> updateWrapper = new UpdateWrapper<>();

 src/main/java/com/dji/sample/media/util/ImageDownloaderAndCompressor.java

@@ -4,6 +4,9 @@
import java.io.*;
import java.net.URL;
import java.util.List;
import java.util.concurrent.CountDownLatch;
import java.util.concurrent.ExecutorService;
import java.util.concurrent.Executors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.imageio.ImageIO;
@@ -12,27 +15,46 @@
public class ImageDownloaderAndCompressor {


    private static final int THREAD_POOL_SIZE = 10;

    public static void downloadAndSaveImages(List<String> urls, String outputFolder) {
        ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
        CountDownLatch latch = new CountDownLatch(urls.size());
        int count = 0;

        for (String imageUrl : urls) {
            try {
                // 下载图片
                URL url = new URL(imageUrl);
                BufferedImage image = ImageIO.read(url);
                // 保存图片
                File outputfile = new File(outputFolder, "image_" + (count++) + ".jpg");
                ImageIO.write(image, "jpg", outputfile);
                System.out.println("Saved: " + outputfile.getAbsolutePath());
            } catch (IOException e) {
                System.err.println("Failed to download image: " + imageUrl);
                e.printStackTrace();
            }
            final int index = count++;
            executorService.submit(() -> {
                try {
                    // 下载图片
                    URL url = new URL(imageUrl);
                    BufferedImage image = ImageIO.read(url);
                    // 保存图片
                    File outputfile = new File(outputFolder, "image_" + index + ".jpg");
                    ImageIO.write(image, "jpg", outputfile);
                    System.out.println("Saved: " + outputfile.getAbsolutePath());
                } catch (IOException e) {
                    System.err.println("Failed to download image: " + imageUrl);
                    e.printStackTrace();
                } finally {
                    latch.countDown();
                }
            });
        }

        try {
            latch.await();
        } catch (InterruptedException e) {
            Thread.currentThread().interrupt();
            System.err.println("Image downloading was interrupted.");
        } finally {
            executorService.shutdown();
        }
    }

    public static void zipAndSendFolder(String folderName, HttpServletResponse response) {
        File zipFile = new File(folderName + ".zip");
        try {
            File zipFile = new File(folderName + ".zip");
            zipFolder(folderName, zipFile.getPath());

            // 设置响应头
@@ -50,12 +72,20 @@
                }
            }

            // 删除临时压缩文件
            zipFile.delete();

        } catch (IOException e) {
            System.err.println("Failed to compress or send folder: " + folderName);
            e.printStackTrace();
            response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
            try {
                response.getWriter().write("Failed to compress or send folder: " + folderName);
            } catch (IOException ex) {
                ex.printStackTrace();
            }
        } finally {
            // 删除临时压缩文件
            if (zipFile.exists()) {
                zipFile.delete();
            }
        }
    }