前端大文件上传

相关参考资料

客户端

import React, { useRef, useEffect, useReducer } from 'react'
import { Button, Progress, message } from 'antd';
import { SlideDown } from 'react-slidedown'
import SparkMD5 from 'spark-md5'
import 'react-slidedown/lib/slidedown.css'
import './style.css'
import axios from 'axios'

// const BaseUrl = 'http://localhost:9094'
const BaseUrl = 'https://jebzkg-gorfvo-9090.preview.cloudstudio.work'

const initialState = { checkPercent: 0, uploadPercent: 0 };

function reducer(state, action) {
  switch (action.type) {
    case 'check':
      initialState.checkPercent = action.checkPercent
      return { ...initialState }
    case 'upload':
      initialState.uploadPercent = action.uploadPercent
      return { ...initialState }
    case 'init':
      initialState.checkPercent = 0
      initialState.uploadPercent = 0
      return { ...initialState }
    default:
      return { checkPercent: state.checkPercent, uploadPercent: state.uploadPercent }
  }
}

const Upload = () => {
  const [state, dispatch] = useReducer(reducer, initialState)
  const inputRef = useRef(null)
  const chunks = 100; // 切成100份
  const chunkSize = 0.15 * 1024 * 1024 // 切片大小
  let checkCurrentChunk = 0; // 检查，当前切片
  let uploadCurrentChunk = 0 // 上传，当前切片

  /**
   * 将文件转换成md5并进行切片
   * @returns md5
   */
  const md5File = (file) => {
    return new Promise((resolve, reject) => {
      // 文件截取
      let blobSlice = File.prototype.slice || File.prototype.mozSlice || File.prototype.webkitSlice,
        chunkSize = file?.size / 100,
        spark = new SparkMD5.ArrayBuffer(),
        fileReader = new FileReader();

      fileReader.onload = function (e) {
        console.log('read chunk nr', checkCurrentChunk + 1, 'of', chunks);
        spark.append(e.target.result);
        checkCurrentChunk += 1;

        if (checkCurrentChunk < chunks) {
          loadNext();
        } else {
          let result = spark.end()
          resolve(result)
        }
      };

      fileReader.onerror = function () {
        message.error('文件读取错误')
      };

      const loadNext = () => {
        const start = checkCurrentChunk * chunkSize,
          end = ((start + chunkSize) >= file.size) ? file.size : start + chunkSize;

        // 文件切片
        fileReader.readAsArrayBuffer(blobSlice.call(file, start, end));
        // 检查进度条
        dispatch({ type: 'check', checkPercent: checkCurrentChunk + 1 })
      }

      loadNext();
    })
    
  }

  /**
   * 校验文件
   * @param {*} fileName 文件名
   * @param {*} fileMd5Value md5文件
   * @returns 
   */
  const checkFileMD5 = (fileName, fileMd5Value) => {
    let url = BaseUrl + '/check/file?fileName=' + fileName + "&fileMd5Value=" + fileMd5Value
    return axios.get(url)
  }

  // 上传chunk
  function upload({ i, file, fileMd5Value, chunks }) {
    // uploadCurrentChunk = 0
    uploadCurrentChunk = i
    //构造一个表单，FormData是HTML5新增的
    let end = (i + 1) * chunkSize >= file.size ? file.size : (i + 1) * chunkSize
    let form = new FormData()
    form.append("data", file.slice(i * chunkSize, end)) //file对象的slice方法用于切出文件的一部分
    form.append("total", chunks) //总片数
    form.append("index", i) //当前是第几片     
    form.append("fileMd5Value", fileMd5Value)
    return axios({
      method: 'post',
      url: BaseUrl + "/upload",
      data: form
    }).then(({ data }) => {
      if (data.stat) {
        uploadCurrentChunk = uploadCurrentChunk + 1
        const uploadPercent = Math.ceil((uploadCurrentChunk / chunks) * 100)
        dispatch({ type: 'upload', uploadPercent })
      }
    })
  }

  /**
   * 上传chunk
   * @param {*} fileMd5Value 
   * @param {*} chunkList 
   */
  async function checkAndUploadChunk(file, fileMd5Value, chunkList) {
    let chunks = Math.ceil(file.size / chunkSize) //总的分片数
    const requestList = []
    for (let i = 0; i < chunks; i++) {
      let exit = chunkList.indexOf(i + "") > -1
      // 如果不存在，则上传
      if (!exit) {
        requestList.push(() => upload({ i, file, fileMd5Value, chunks }))
      }
    }

    // 并发上传
    //3个并发
    if (requestList?.length) {
      // await Promise.all(requestList)
      while(requestList.length){
        const taskArr = requestList.splice(0, 3);
        for(let task of taskArr.map((_task) => _task())){
          await task;
        }
        // await Promise.all(cbs[0](), cbs[1]());
      }
    }
  }

  // 已读取好本地文件
  const responseChange = async (file) => {
    // 1.校验文件，返回md5
    const fileMd5Value = await md5File(file)
    // 2.校验文件的md5
    const { data } = await checkFileMD5(file.name, fileMd5Value)
    // 如果文件已存在, 就秒传
    if (data?.file) {
      message.success('文件已秒传')
      return
    }
    // 3：检查并上传切片
    await checkAndUploadChunk(file, fileMd5Value, data.chunkList)
    // 4：通知服务器所有服务器分片已经上传完成
    notifyServer(file, fileMd5Value)
  }

  /**
   * 所有的分片上传完成，准备合成
   * @param {*} file 
   * @param {*} fileMd5Value 
   */
  function notifyServer(file, fileMd5Value) {
    let url = BaseUrl + '/merge?md5=' + fileMd5Value + "&fileName=" + file.name + "&size=" + file.size
    axios.get(url).then(({ data }) => {
      if (data.stat) {
        message.success('上传成功')
      } else {
        message.error('上传失败')
      }
    })
  }

  useEffect(() => {
    const changeFile = ({ target }) => {
      dispatch({ type: 'init' })
      const file = target.files[0]
      responseChange(file)
    }

    document.addEventListener("change", changeFile)

    return () => {
      document.removeEventListener("change", changeFile)
    }
  // eslint-disable-next-line react-hooks/exhaustive-deps
  }, [])

  return (
    <div className="wrap">
      <div className="upload">
        <span>点击上传文件：</span>
        <input ref={inputRef} type="file" id="file" />
        <Button type="primary" onClick={() => inputRef.current.click()}>上传</Button>
      </div>
      {state.checkPercent > 0 && (
        <SlideDown className={'my-dropdown-slidedown'} >
          <div className="uploading">
            <div>校验文件进度：<Progress style={{ width: 200 }} percent={state.checkPercent} /></div>
          </div>
        </SlideDown>
      )}
      {state.uploadPercent > 0 && (
        <SlideDown className={'my-dropdown-slidedown'} >
          <div className="uploading">
            上传文件进度：<Progress type="circle" percent={state.uploadPercent} />
          </div>
        </SlideDown>
      )}
    </div >
  )
}

export default Upload

服务端

let express = require('express')
let app = express()
let formidable = require('formidable')
let path = require('path')
let uploadDir = 'nodeServer/uploads'
let fs = require('fs-extra')
let concat = require('concat-files')

// 处理跨域
app.all('*', (req, res, next) => {
  res.header('Access-Control-Allow-Origin', '*')
  res.header(
      'Access-Control-Allow-Headers',
      'Content-Type,Content-Length, Authorization, Accept,X-Requested-With'
  )
  res.header('Access-Control-Allow-Methods', 'PUT,POST,GET,DELETE,OPTIONS')
  res.header('X-Powered-By', ' 3.2.1')
  if (req.method === 'OPTIONS') res.send(200) /*让options请求快速返回*/
  else next()
})

// 列出文件夹下所有文件
function listDir(path) {
  return new Promise((resolve, reject) => {
      fs.readdir(path, (err, data) => {
          if (err) {
              reject(err)
              return
          }
          // 把mac系统下的临时文件去掉
          if (data && data.length > 0 && data[0] === '.DS_Store') {
              data.splice(0, 1)
          }
          resolve(data)
      })
  })
}

// 文件或文件夹是否存在
function isExist(filePath) {
  return new Promise((resolve, reject) => {
      fs.stat(filePath, (err, stats) => {
          // 文件不存在
          if (err && err.code === 'ENOENT') {
              resolve(false)
          } else {
              resolve(true)
          }
      })
  })
}

// 获取文件Chunk列表
async function getChunkList(filePath, folderPath, callback) {
  let isFileExit = await isExist(filePath)
  let result = {}
  // 如果文件已在存在, 不用再继续上传, 真接秒传
  if (isFileExit) {
      result = {
          stat: 1,
          file: {
              isExist: true,
              name: filePath
          },
          desc: 'file is exist'
      }
  } else {
      let isFolderExist = await isExist(folderPath)
      // 如果文件夹(md5值后的文件)存在, 就获取已经上传的块
      let fileList = []
      if (isFolderExist) {
          fileList = await listDir(folderPath)
      }
      result = {
          stat: 1,
          chunkList: fileList,
          desc: 'folder list'
      }
  }
  callback(result)
}

/**
 * 检查md5
 */
app.get('/check/file', (req, resp) => {
  let query = req.query
  let fileName = query.fileName
  let fileMd5Value = query.fileMd5Value
  // 获取文件Chunk列表
  getChunkList(
      path.join(uploadDir, fileName),
      path.join(uploadDir, fileMd5Value),
      data => {
          resp.send(data)
      }
  )
})
// app.get('/check/file', (req, resp) => {
//   resp.send('ok!');
// })

// 合并文件
async function mergeFiles(srcDir, targetDir, newFileName) {
    let fileArr = await listDir(srcDir)
    fileArr.sort((x,y) => {
        return x-y;
    })
    // 把文件名加上文件夹的前缀
    for (let i = 0; i < fileArr.length; i++) {
        fileArr[i] = srcDir + '/' + fileArr[i]
    }
    concat(fileArr, path.join(targetDir, newFileName), err => {
        if(err) {
            return false
        }
        return true
    })
  }
  
  // 合成
  app.get('/merge', (req, resp) => {
    let query = req.query
    let md5 = query.md5
    let fileName = query.fileName
    const res = mergeFiles(path.join(uploadDir, md5), uploadDir, fileName)
    resp.send({
        stat: res ? 1 : 0
    })
  })

app.post('/upload', (req, resp) => {
  const form = new formidable.IncomingForm({
      uploadDir: 'nodeServer/tmp'
  })
  form.parse(req, function(err, fields, file) {
      let index = fields.index
      let fileMd5Value = fields.fileMd5Value
      let folder = path.resolve(__dirname, 'nodeServer/uploads', fileMd5Value)
      folderIsExit(folder).then(val => {
          let destFile = path.resolve(folder, fields.index)
          copyFile(file.data.path, destFile).then(
              successLog => {
                  resp.send({
                      stat: 1,
                      desc: index
                  })
              },
              errorLog => {
                  resp.send({
                      stat: 0,
                      desc: 'Error'
                  })
              }
          )
      })
  })
  
  // 文件夹是否存在, 不存在则创建文件
  function folderIsExit(folder) {
      return new Promise(async (resolve, reject) => {
          await fs.ensureDirSync(path.join(folder))
          resolve(true)
      })
  }
  // 把文件从一个目录拷贝到别一个目录
  function copyFile(src, dest) {
      let promise = new Promise((resolve, reject) => {
          fs.rename(src, dest, err => {
              if (err) {
                  reject(err)
              } else {
                  resolve('copy file:' + dest + ' success!')
              }
          })
      })
      return promise
  }
})

app.listen(9090, () => {
  console.log('服务启动完成，端口监听9090！')
})

效果图