Springboot 分片上传

核心逻辑

基于new RandomAccessFile(dest, “rw”)类处理，前端会上传一系列的分片数据，里面包含了被分片文件的总大小根据这个文件的总大小通过RandomAccessFile创建一个空内容的文件读取每一次上传的分片文件大小和当前属于第几片，通过算法得到起始位置后，调用RandomAccessFile的seek方法移动游标读取分片文件的数据并写入（randomAccessFile.write）

源码

DTO

import lombok.Data; import lombok.experimental.Accessors; import org.springframework.web.multipart.MultipartFile; /** * @author zhe.xiao * @date 2020/10/21 */ @Data @Accessors(chain = true) public class FileUploadDTO { //文件名 private String name; //md5码 private String md5; //文件总大小 private Long size; //文件一共被分了多少个片 private Integer chunks; //文件当前是第几个分片（第一个分片从0开始） private Integer chunk; //文件的数据 private MultipartFile file; }

VO

import lombok.Data; import lombok.experimental.Accessors; /** * @author zhe.xiao * @date 2020/10/21 */ @Data @Accessors(chain = true) public class FileUploadVO { private String md5; private String filename; private String filepath; }

FileUploadUtils

/** * @author zhe.xiao * @date 2020/10/21 */ public class FileUploadUtils { private static final Logger logger = LoggerFactory.getLogger(FileUploadUtils.class); private static String uploadFolder = "./upload"; /** * key = md5 * value = ChunkInfo * <p> * 通过md5记录文件分片数据 */ private static Map<String, ChunkInfo> fileMap = new HashMap<>(); /** * 生成一个与文件md5关联的数据信息 * <p> * 根据md5生成一个唯一的文件名和每个chunk的初始上传状态false * * @param uploadDTO * @return */ public static String getFilename(FileUploadDTO uploadDTO) { String md5 = uploadDTO.getMd5(); if (!checkFile(md5)) { synchronized (FileUploadUtils.class) { if (!checkFile(md5)) { fileMap.put(md5, new ChunkInfo(uploadDTO.getName(), uploadDTO.getChunks())); } } } return fileMap.get(md5).uniqueName; } /** * 判断文件是否有分块 * * @param md5 * @return */ public static Boolean checkFile(String md5) { return fileMap.containsKey(md5); } /** * 为文件的某个chunk添加上传完毕的分块记录 * * @param md5 md5值 * @param chunk 分块号 */ public static void addFileChunk(String md5, int chunk) { fileMap.get(md5).chunkStatus[chunk] = true; } /** * 删除文件信息 * * @param md5 */ public static void removeFile(String md5) { if (checkFile(md5)) { fileMap.remove(md5); } } /** * 判断文件所有分块是否已上传完毕 * @param md5 * @return */ public static boolean finished(String md5) { if (checkFile(md5)) { for (Boolean done : fileMap.get(md5).chunkStatus) { if (null == done || !done) { return false; } } return true; } return false; } /** * 上传文件分片 * * @param uploadDTO */ public static FileUploadVO uploadChunk(FileUploadDTO uploadDTO) throws Exception { FileUploadVO uploadVO = saveChunk(uploadDTO); //指定某个chunk上传完毕 String md5 = uploadDTO.getMd5(); addFileChunk(md5, uploadDTO.getChunk()); if(finished(md5)){ logger.info("==================="); logger.info("uploadChunk finished fileMap="+fileMap.toString()); removeFile(md5); logger.info("uploadChunk finished fileMap="+fileMap.toString()); logger.info("==================="); } return uploadVO; } /** * 保存文件分片 * @param uploadDTO * @throws Exception */ private static FileUploadVO saveChunk(FileUploadDTO uploadDTO) throws Exception{ //文件路径 String filename = getFilename(uploadDTO); String dest = uploadFolder + "/" + filename; //得到文件信息 MultipartFile file = uploadDTO.getFile(); if (null == file) { throw new RuntimeException("缺少file数据"); } InputStream fis = file.getInputStream(); Long fileSize = uploadDTO.getSize(); Long chunkSize = file.getSize(); //得到总分片数和当前分片 Integer chunks = uploadDTO.getChunks(); Integer chunk = uploadDTO.getChunk(); logger.info("==================="); logger.info("saveChunk filename="+filename); logger.info("saveChunk dest="+dest); logger.info("saveChunk chunk="+chunk); logger.info("saveChunk md5="+uploadDTO.getMd5()); logger.info("==================="); //文件上传 RandomAccessFile randomAccessFile = new RandomAccessFile(dest, "rw"); randomAccessFile.setLength(fileSize); //最后一个分片的size直接获得差值，其余的通过当前分片*size即可（工作条件是除最后一个分片外，每个分片size大小一致） if (chunk == chunks - 1 && chunk != 0) { randomAccessFile.seek(chunk * (fileSize - chunkSize) / chunk); } else { randomAccessFile.seek(chunk * chunkSize); } //保存数据 byte[] buf = new byte[1024]; int len; while (-1 != (len = fis.read(buf))) { randomAccessFile.write(buf, 0, len); } randomAccessFile.close(); //返回的数据 FileUploadVO uploadVO = new FileUploadVO(); uploadVO.setFilename(filename).setMd5(uploadDTO.getMd5()).setFilepath(dest); logger.info("==================="); logger.info("saveChunk uploadVO="+uploadVO); logger.info("==================="); return uploadVO; } /** * 生成随机文件名 * * 不要加后缀，加了后缀后会造成文件被额外的进程使用 * * @return */ public static String generateFilename(String filename) { // String suffix = filename.substring(filename.lastIndexOf(".") + 1); return UUID.randomUUID().toString().replace("-", ""); } /** * 内部类记录分块上传文件信息 */ private static class ChunkInfo { //md5确定文件的唯一名称 String uniqueName; //记录每个分块的状态，上传完毕与否 Boolean[] chunkStatus; ChunkInfo(String oldFilename, Integer chunks) { this.uniqueName = generateFilename(oldFilename); this.chunkStatus = new Boolean[chunks]; } } }

调用

FileUploadVO uploadVO = FileUploadUtils.uploadChunk(uploadDTO);

前端测试数据

断点续传原理测试

public class randomAccessTest { private static Long position = -1L; public static void main(String[] args) throws Exception { // 源文件与目标文件 File sourceFile = new File("./t1.txt"); File targetFile = new File("./t2.txt"); // 输入输出流 FileInputStream fis = null; FileOutputStream fos = null; // 数据缓冲区，每次读一个字节 byte[] buf = new byte[1]; try { fis = new FileInputStream(sourceFile); fos = new FileOutputStream(targetFile); // 数据读写 int len = -1; while (-1 != (len = fis.read(buf))) { fos.write(buf, 0, len); System.out.println(sourceFile.length()); System.out.println(targetFile.length()); System.out.println("======================"); // 当已经上传了3字节的文件内容时，网络中断了，抛出异常 // length返回的是当前写入到了第几个字节 // 只写入了 abc 即返回 3 if (targetFile.length() == 5) { position = targetFile.length(); throw new FileAccessException(); } } } catch (FileAccessException e) { keepGoing(sourceFile, targetFile, position); } catch (Exception e) { System.out.println(e.getMessage()); } finally { assert fis != null; fis.close(); assert fos != null; fos.close(); } } private static void keepGoing(File source, File target, Long position) throws Exception { System.out.println("记录position="+position + "，开始断点续传，5秒后文件会写入完毕"); Thread.sleep(5000); RandomAccessFile readFile = new RandomAccessFile(source, "rw"); RandomAccessFile writeFile = new RandomAccessFile(target, "rw"); //seek是从0开始的，position=3即从第四个值做为起始点 d 开始读取 readFile.seek(position); writeFile.seek(position); // 数据缓冲区 byte[] buf = new byte[1]; int len; while ((len = readFile.read(buf)) != -1) { writeFile.write(buf, 0, len); } readFile.close(); writeFile.close(); } } class FileAccessException extends Exception { }

分片合并测试

/** * @author zhe.xiao * @date 2020/10/22 */ public class smallFileMerge { public static void main(String[] args) { String[] strings = {"./t1.txt", "./t2.txt"}; String resultPath = "./result.txt"; mergeFiles(strings, resultPath); } public static boolean mergeFiles(String[] fpaths, String resultPath) { if (fpaths == null || fpaths.length < 1) { return false; } if (fpaths.length == 1) { return new File(fpaths[0]).renameTo(new File(resultPath)); } File[] files = new File[fpaths.length]; for (int i = 0; i < fpaths.length; i ++) { files[i] = new File(fpaths[i]); if (!files[i].exists() || !files[i].isFile()) { return false; } } File resultFile = new File(resultPath); try { FileChannel resultFileChannel = new FileOutputStream(resultFile, true).getChannel(); for (int i = 0; i < fpaths.length; i ++) { FileChannel blk = new FileInputStream(files[i]).getChannel(); resultFileChannel.transferFrom(blk, resultFileChannel.size(), blk.size()); blk.close(); } resultFileChannel.close(); } catch (IOException e) { e.printStackTrace(); return false; } for (int i = 0; i < fpaths.length; i ++) { files[i].delete(); } return true; } }