From bf7a9ef439e3b49268974d564107999619b2b4fd Mon Sep 17 00:00:00 2001
From: rain <167982779@qq.com>
Date: Sat, 20 Jul 2024 09:28:40 +0800
Subject: [PATCH] 优化图片压缩下载使用多线程
---
src/main/java/com/dji/sample/media/util/ImageDownloaderAndCompressor.java | 62 +++++++++++++++-----
src/main/java/com/dji/sample/media/service/impl/FileServiceImpl.java | 92 ++++++++++++++++++++++--------
src/main/java/com/dji/sample/media/controller/FileController.java | 16 -----
src/main/java/com/dji/sample/media/service/IFileService.java | 4 +
src/main/java/com/dji/sample/media/dao/IFileMapper.java | 2
5 files changed, 120 insertions(+), 56 deletions(-)
diff --git a/src/main/java/com/dji/sample/media/controller/FileController.java b/src/main/java/com/dji/sample/media/controller/FileController.java
index 5200eea..8a0093f 100644
--- a/src/main/java/com/dji/sample/media/controller/FileController.java
+++ b/src/main/java/com/dji/sample/media/controller/FileController.java
@@ -171,23 +171,9 @@
}
}
-
- private static final String BASE_URL = "http://dev.jxpskj.com:9000/cloud-bucket";
-
@GetMapping("/download-images")
public void downloadImages(@RequestParam String jobId, HttpServletResponse response) {
- List<MediaFileEntity> entities = fileService.getMedia(jobId);
- List<String> urls = entities.stream()
- .map(MediaFileEntity::getObjectKey)
- .map(objectKey -> BASE_URL + objectKey)
- .collect(Collectors.toList());
-
- String outputFolder = "images"; // 存放图片的文件夹
- new File(outputFolder).mkdirs(); // 创建文件夹
-
- downloadAndSaveImages(urls, outputFolder);
- zipAndSendFolder(outputFolder, response);
- cleanUp(outputFolder);
+ fileService.downloadImages(jobId,response);
}
}
\ No newline at end of file
diff --git a/src/main/java/com/dji/sample/media/dao/IFileMapper.java b/src/main/java/com/dji/sample/media/dao/IFileMapper.java
index d1e773f..5886478 100644
--- a/src/main/java/com/dji/sample/media/dao/IFileMapper.java
+++ b/src/main/java/com/dji/sample/media/dao/IFileMapper.java
@@ -8,6 +8,7 @@
import com.dji.sample.media.model.MediaJobDTO;
import com.dji.sample.media.model.param.SearchMediaParam;
import com.dji.sample.wayline.model.dto.WaylineJobDTO;
+import org.apache.ibatis.annotations.Mapper;
import org.apache.ibatis.annotations.Param;
/**
@@ -15,6 +16,7 @@
* @version 0.2
* @date 2021/12/9
*/
+@Mapper
public interface IFileMapper extends BaseMapper<MediaFileEntity> {
Page<MediaFileEntity> getPage(@Param("page") Page<MediaFileEntity> mediaFileEntityPage, @Param("workspaceId") String workspaceId,@Param("query") MediaFileQueryParam mediaFileQueryParam);
diff --git a/src/main/java/com/dji/sample/media/service/IFileService.java b/src/main/java/com/dji/sample/media/service/IFileService.java
index 5e0389f..41eb659 100644
--- a/src/main/java/com/dji/sample/media/service/IFileService.java
+++ b/src/main/java/com/dji/sample/media/service/IFileService.java
@@ -5,7 +5,9 @@
import com.dji.sample.common.model.ResponseResult;
import com.dji.sample.media.model.*;
import com.drew.imaging.ImageProcessingException;
+import org.springframework.web.bind.annotation.RequestParam;
+import javax.servlet.http.HttpServletResponse;
import java.awt.*;
import java.io.IOException;
import java.net.URL;
@@ -102,6 +104,8 @@
*/
URL getObjectUrl(String workspaceId, String fileId);
+ void downloadImages( String jobId, HttpServletResponse response);
+
/**
* Query all media files of a job.
*
diff --git a/src/main/java/com/dji/sample/media/service/impl/FileServiceImpl.java b/src/main/java/com/dji/sample/media/service/impl/FileServiceImpl.java
index 6b52db1..3b944e5 100644
--- a/src/main/java/com/dji/sample/media/service/impl/FileServiceImpl.java
+++ b/src/main/java/com/dji/sample/media/service/impl/FileServiceImpl.java
@@ -43,6 +43,7 @@
import org.springframework.stereotype.Service;
import org.springframework.transaction.annotation.Transactional;
+import javax.servlet.http.HttpServletResponse;
import java.awt.*;
import java.io.File;
import java.io.FileInputStream;
@@ -54,6 +55,8 @@
import java.util.*;
import java.util.List;
import java.util.stream.Collectors;
+
+import static com.dji.sample.media.util.ImageDownloaderAndCompressor.*;
/**
* @author sean
@@ -429,6 +432,27 @@
}
@Override
+ public void downloadImages(String jobId, HttpServletResponse response) {
+ String address = pojo.getEndpoint() + "/" + pojo.getBucket();
+ List<MediaFileEntity> entities = getMedia(jobId);
+ List<String> urls = entities.stream()
+ .map(MediaFileEntity::getObjectKey)
+ .map(objectKey -> address + objectKey)
+ .collect(Collectors.toList());
+ String outputFolder = "images"; // 存放图片的文件夹
+ new File(outputFolder).mkdirs(); // 创建文件夹
+ try {
+ downloadAndSaveImages(urls, outputFolder);
+ zipAndSendFolder(outputFolder, response);
+ } catch (Exception e) {
+ e.printStackTrace();
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ } finally {
+ cleanUp(outputFolder);
+ }
+ }
+
+ @Override
public List<MediaFileDTO> getFilesByWorkspaceAndJobId(String workspaceId, String jobId) {
return mapper.selectList(new LambdaQueryWrapper<MediaFileEntity>()
.eq(MediaFileEntity::getWorkspaceId, workspaceId)
@@ -791,43 +815,58 @@
}
@Override
public PaginationData<MediaFileEntity> getJobId(int pageNum, int pageSize, String workspaceId) {
- // 分页查询
- Page<MediaFileEntity> page = new Page<>(pageNum, pageSize);
- Page<MediaFileEntity> resultPage = mapper.selectPage(page, new LambdaQueryWrapper<MediaFileEntity>()
- .eq(MediaFileEntity::getWorkspaceId, workspaceId));
+ List<MediaFileEntity> allRecords = new ArrayList<>();
+ int currentPageNum = 1; // 从第一页开始获取记录
- List<MediaFileEntity> result = resultPage.getRecords();
+ // 用于存储去重后的记录
+ Map<String, MediaFileEntity> uniqueFilesMap = new HashMap<>();
- // 根据 fileName 字段进行去重并设置 JobName
- Map<String, MediaFileEntity> uniqueFilesMap = result.stream()
- .peek(mediaFile -> {
- String taskNameResult = waylineJobService.getName(mediaFile.getJobId());
- mediaFile.setJobName(taskNameResult);
- })
- .collect(Collectors.toMap(
- MediaFileEntity::getFileName,
- entity -> entity,
- (existing, replacement) -> existing // 如果有重复的，保留已有的
- ));
+ while (true) {
+ // 分页查询
+ Page<MediaFileEntity> page = new Page<>(currentPageNum, pageSize);
+ Page<MediaFileEntity> resultPage = mapper.selectPage(page, new LambdaQueryWrapper<MediaFileEntity>()
+ .eq(MediaFileEntity::getWorkspaceId, workspaceId));
+ List<MediaFileEntity> result = resultPage.getRecords();
+
+ // 检查是否还有更多记录
+ if (result.isEmpty()) {
+ break; // 没有更多记录了，停止请求
+ }
+
+ // 根据 fileName 字段进行去重并设置 JobName
+ result.stream()
+ .peek(mediaFile -> {
+ String taskNameResult = waylineJobService.getName(mediaFile.getJobId());
+ mediaFile.setJobName(taskNameResult);
+ })
+ .forEach(mediaFile -> uniqueFilesMap.putIfAbsent(mediaFile.getFileName(), mediaFile));
+
+ // 增加当前页数以获取更多记录
+ currentPageNum++;
+ }
+
+ // 获取去重后的实际总数
List<MediaFileEntity> uniqueFiles = new ArrayList<>(uniqueFilesMap.values());
-
- // 使用去重后的列表重新计算总数
int uniqueTotal = uniqueFiles.size();
+
+ // 计算当前页的起始和结束索引
+ int fromIndex = Math.min((pageNum - 1) * pageSize, uniqueTotal);
+ int toIndex = Math.min(fromIndex + pageSize, uniqueTotal);
+
+ // 获取当前页的数据
+ List<MediaFileEntity> pagedUniqueFiles = uniqueFiles.subList(fromIndex, toIndex);
// 创建新的分页对象，使用去重后的总数
Page<MediaFileEntity> uniquePage = new Page<>(pageNum, pageSize, uniqueTotal);
-
- // 截取当前页的数据
- int fromIndex = Math.min((pageNum - 1) * pageSize, uniqueTotal);
- int toIndex = Math.min(fromIndex + pageSize, uniqueTotal);
- List<MediaFileEntity> pagedUniqueFiles = uniqueFiles.subList(fromIndex, toIndex);
// 使用新的分页对象构造 Pagination
Pagination pagination = new Pagination(uniquePage);
return new PaginationData<>(pagedUniqueFiles, pagination);
}
+
+
@Override
@@ -870,9 +909,12 @@
- public List<MediaFileEntity> getMedia(String jobId){
- return mapper.selectList(new LambdaQueryWrapper<MediaFileEntity>().eq(MediaFileEntity::getJobId,jobId));
+ public List<MediaFileEntity> getMedia(String jobId) {
+ return mapper.selectList(new LambdaQueryWrapper<MediaFileEntity>()
+ .eq(MediaFileEntity::getJobId, jobId)
+ .groupBy(MediaFileEntity::getFileName)); // 使用groupBy去重
}
+
public void updateById(Integer id, MediaFileMarkEntity entity) {
entity.setIsadd(1);
UpdateWrapper<MediaFileMarkEntity> updateWrapper = new UpdateWrapper<>();
diff --git a/src/main/java/com/dji/sample/media/util/ImageDownloaderAndCompressor.java b/src/main/java/com/dji/sample/media/util/ImageDownloaderAndCompressor.java
index d5f98b8..3818a95 100644
--- a/src/main/java/com/dji/sample/media/util/ImageDownloaderAndCompressor.java
+++ b/src/main/java/com/dji/sample/media/util/ImageDownloaderAndCompressor.java
@@ -4,6 +4,9 @@
import java.io.*;
import java.net.URL;
import java.util.List;
+import java.util.concurrent.CountDownLatch;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.Executors;
import java.util.zip.ZipEntry;
import java.util.zip.ZipOutputStream;
import javax.imageio.ImageIO;
@@ -12,27 +15,46 @@
public class ImageDownloaderAndCompressor {
+ private static final int THREAD_POOL_SIZE = 10;
+
public static void downloadAndSaveImages(List<String> urls, String outputFolder) {
+ ExecutorService executorService = Executors.newFixedThreadPool(THREAD_POOL_SIZE);
+ CountDownLatch latch = new CountDownLatch(urls.size());
int count = 0;
+
for (String imageUrl : urls) {
- try {
- // 下载图片
- URL url = new URL(imageUrl);
- BufferedImage image = ImageIO.read(url);
- // 保存图片
- File outputfile = new File(outputFolder, "image_" + (count++) + ".jpg");
- ImageIO.write(image, "jpg", outputfile);
- System.out.println("Saved: " + outputfile.getAbsolutePath());
- } catch (IOException e) {
- System.err.println("Failed to download image: " + imageUrl);
- e.printStackTrace();
- }
+ final int index = count++;
+ executorService.submit(() -> {
+ try {
+ // 下载图片
+ URL url = new URL(imageUrl);
+ BufferedImage image = ImageIO.read(url);
+ // 保存图片
+ File outputfile = new File(outputFolder, "image_" + index + ".jpg");
+ ImageIO.write(image, "jpg", outputfile);
+ System.out.println("Saved: " + outputfile.getAbsolutePath());
+ } catch (IOException e) {
+ System.err.println("Failed to download image: " + imageUrl);
+ e.printStackTrace();
+ } finally {
+ latch.countDown();
+ }
+ });
+ }
+
+ try {
+ latch.await();
+ } catch (InterruptedException e) {
+ Thread.currentThread().interrupt();
+ System.err.println("Image downloading was interrupted.");
+ } finally {
+ executorService.shutdown();
}
}
public static void zipAndSendFolder(String folderName, HttpServletResponse response) {
+ File zipFile = new File(folderName + ".zip");
try {
- File zipFile = new File(folderName + ".zip");
zipFolder(folderName, zipFile.getPath());
// 设置响应头
@@ -50,12 +72,20 @@
}
}
- // 删除临时压缩文件
- zipFile.delete();
-
} catch (IOException e) {
System.err.println("Failed to compress or send folder: " + folderName);
e.printStackTrace();
+ response.setStatus(HttpServletResponse.SC_INTERNAL_SERVER_ERROR);
+ try {
+ response.getWriter().write("Failed to compress or send folder: " + folderName);
+ } catch (IOException ex) {
+ ex.printStackTrace();
+ }
+ } finally {
+ // 删除临时压缩文件
+ if (zipFile.exists()) {
+ zipFile.delete();
+ }
}
}
--
Gitblit v1.9.3