add Parallel upload and GZIP compression

[yanzhangshuai]

I made 3 improvements to the code

1. Parallel uploads.
2. Retry
3. GZIP compression

import color from 'picocolors'
import fg from 'fast-glob'
import path from 'path'
import OSS from 'ali-oss'
import { URL } from 'node:url'
import zlib from 'zlib'
import fs from 'fs-extra'

import { normalizePath, type Plugin, type ResolvedBuildOptions } from "vite";



const DEF_GZIP = true

const RETRY_COUNT = 3



export interface vitePluginAliOssOptions {
  retry?:           number
  gzip?:            boolean | number
  enabled?:         boolean
  region:           string
  accessKeyId:      string
  accessKeySecret:  string
  bucket:           string
  overwrite?:       boolean
  ignore?:          string[]
  headers?:         Record<string, string>
  test?:            boolean
}
export default function vitePluginAliOss(options: vitePluginAliOssOptions): Plugin {
  let baseConfig = '/'
  let buildConfig: ResolvedBuildOptions

  return {
    name: 'vite-plugin-ali-oss',
    enforce: 'post',
    apply: 'build',
    configResolved(config) {
      baseConfig = config.base
      buildConfig = config.build
    },

    closeBundle: {
      sequential: true,
      order: 'post',
      async handler() {
        if (!options.enabled) {
          return
        }

        if (!/^http/i.test(baseConfig)) {
          throw Error('[vite-plugin-ali-oss] base must be a url')
        }

        const outDirPath = normalizePath(path.resolve(normalizePath(buildConfig.outDir)))


        options.gzip  = options?.gzip === undefined ? DEF_GZIP : options.gzip
        options.retry = options?.retry === undefined || options?.retry < 0 ? RETRY_COUNT : options.retry

        const { pathname: ossBasePath, origin: ossOrigin } = new URL(baseConfig)
        const ssrClient = buildConfig.ssrManifest
        const ssrServer = buildConfig.ssr


        const client = new OSS({
          region:           options.region,
          accessKeyId:      options.accessKeyId,
          accessKeySecret:  options.accessKeySecret,
          bucket:           options.bucket
        })
       

        const files = await fg(
          outDirPath + '/**/*',
          {
            dot: true,
            ignore:
              // custom ignore
              options.ignore !== undefined ? options.ignore :
                // ssr client ignore
                ssrClient ? ['**/ssr-manifest.json', '**/*.html'] :
                  // ssr server ignore
                  ssrServer ? ['**'] :
                    // default ignore
                    ['**/*.html']
          }
        )


        console.log('')
        console.log('ali oss upload start' + (ssrClient ? ' (ssr client)' : ssrServer ? ' (ssr server)' : ''))
        console.log('')

        const upload = async (ossFilePath: string, filePath: string, headers: Record<string, string | number | boolean> = {}) => {

          headers = Object.assign({}, options.headers || {}, headers || {})

          const output = `${buildConfig.outDir + filePath} => ${color.green(ossOrigin + ossFilePath)}`

          const fn = (file: string | Buffer, retryCount = 0) => {

            return new Promise((resolve, reject) => {

              client.put(ossFilePath, file, { headers })
              .then(res => {
                console.log(`ali oss ${color.green('upload success')}: ${output}`)
                resolve(res)
              })
              .catch(err => {
                if (retryCount < options.retry!) {
                  console.log(`${color.yellow(`retry ${retryCount + 1}`)} upload ${output}`)
                  return fn(file, retryCount + 1)
                }

                reject(err) 
              })
            })

          }


          if (!options.gzip) {
            return fn(filePath)
          }

          const file = await getFile(filePath, options.gzip)
          headers['Content-Encoding'] = 'gzip'
          return fn(file)
        }

        const startTime = new Date().getTime()

        const req = files.map(async f => {
          const filePath = normalizePath(f).split(outDirPath)[1] // eg: '/assets/vendor.bfb92b77.js'
          const ossFilePath = ossBasePath.replace(/\/$/, '') + filePath // eg: '/base/assets/vendor.bfb92b77.js'
          const output = `${buildConfig.outDir + filePath} => ${color.green(ossOrigin + ossFilePath)}`

          try {
            if (options.test) {
              console.log(`test upload path: ${output}`)
              return Promise.resolve()
            }

            if (options.overwrite) {
              return upload(ossFilePath, f)
            }

            await client.head(ossFilePath);
            console.log(`ali oss ${color.gray('files exists')}: ${output}`)
            return Promise.resolve()

          } catch (error: any) {
            if (error.code === 'NoSuchKey') {
              return upload(ossFilePath, f, { 'x-oss-forbid-overwrite': true })

            } else {
              throw new Error(error)
            }
          }

        })


        try {

          await Promise.all(req)

          const duration = (new Date().getTime() - startTime) / 1000

          console.log('')
          console.log(`ali oss upload complete ^_^, cost ${duration.toFixed(2)}s`)
          console.log('')
        } catch (error) {
          console.log('')
          console.log(`ali oss ${color.red('upload fail')}` + color.red(`, error: ${error}`))
          console.log('')
        }
      }
    }
  }
}

[xiaweiss]

感谢贡献

1. Parallel uploads 服务器的带宽是有限的，并发上传不一定缩短时间，我需要测试看看

2. Retry 重试的 case，不太容易测试，我生产环境也没遇到过这类报错。粗看代码的话，重试后没有 resolve，这块的逻辑我需要花时间再看看

3. GZIP compression

oss 只需要下载时开启 gzip 即可，上传并不需要吧？ https://www.alibabacloud.com/help/zh/oss/user-guide/how-do-i-compress-objects-that-i-download-from-oss-in-the-gzip-format

能否提供更多 gzip 相关细节？感谢🙏

[xiaweiss]

根据以下资料，可能需要本地进行 gzip 压缩操作

https://github.com/ali-sdk/ali-oss/issues/555 https://github.com/ali-oss-gzip/ali-oss-gzip/blob/master/lib/uploader.js#L45

[yanzhangshuai]

感谢贡献

1. Parallel uploads 服务器的带宽是有限的，并发上传不一定缩短时间，我需要测试看看
2. Retry 重试的 case，不太容易测试，我生产环境也没遇到过这类报错。粗看代码的话，重试后没有 resolve，这块的逻辑我需要花时间再看看
3. GZIP compression

oss 只需要下载时开启 gzip 即可，上传并不需要吧？ https://www.alibabacloud.com/help/zh/oss/user-guide/how-do-i-compress-objects-that-i-download-from-oss-in-the-gzip-format

能否提供更多 gzip 相关细节？感谢🙏

1. 并行上传我这里速度提升是成倍级别的，大概我这里只上传代码。

2. 重试的确不太容易测试，我这里测试了两天，也仅碰到过几次，重试的代码是有resolve的，就是为了resolve，才用了new Promise, await就不行了。

3. gzip,压缩是为了提高上传速度，

这三个功能来源于我之前使用的webpack插件 webpack-alioss-plugin

[xiaweiss]

好的，我抽空看看

[xiaweiss]

@yanzhangshuai 根据目前的测试结果来看，无需任何设置，gzip 是默认开启的

https://help.aliyun.com/zh/oss/user-guide/how-do-i-compress-objects-that-i-download-from-oss-in-the-gzip-format 根据官方文档的说明，只需要 GET 请求在下载时设置 Content-Encoding: gzip 即可

目前打包时，不传任何的请求头，结果就是 gzip 的，见下图：

如果 oss 上传时手动设置 headers，结果是文件损坏无法打开，见下图：

如果你有传 header 的需求，可以通过本插件的 headers 选项传递

[xiaweiss]

retry 选项在 issue #17 跟进，本周会发布一个版本