diff --git a/README.md b/README.md index c286839..3d4a0d4 100644 --- a/README.md +++ b/README.md @@ -29,15 +29,16 @@ All following options except `config` and `config-key` can be configured using t Available AWS regions can be viewed at https://docs.aws.amazon.com/sns/latest/dg/sns-supported-regions-countries.html. ``` ---config, -c path to config file [default: "./ucdn.yml"] ---config-key, -C root config key [default: null] ---region, -r AWS geographical area [default: "eu-west-1"] ---dir, -d assets directory [default: "dist/"] ---bucket, -b AWS bucket for upload [required] ---exclude, -e excluded extenstions [default: ["html","gz"]] ---accessKeyId, --access-key-id AWS access key ID [required] ---secretAccessKey, --secret-access-key AWS secret access key [required] ---targetDir, --target-dir AWS bucket target directory [default: ""] +--config, -c path to config file [default: "./ucdn.yml"] +--config-key, -C root config key [default: null] +--region, -r AWS geographical area [default: "eu-west-1"] +--dir, -d assets directory [default: "dist/"] +--bucket, -b AWS bucket for upload [required] +--exclude, -e excluded extenstions [default: ["html","gz"]] +--accessKeyId, --access-key-id AWS access key ID [required] +--secretAccessKey, --secret-access-key AWS secret access key [required] +--targetDir, --target-dir AWS bucket target directory [default: ""] +--concurrencyLimit, --processes Limit of concurrent upload processes [default: 100] ``` ## Contributing diff --git a/package.json b/package.json index 20e2a33..6de7d55 100644 --- a/package.json +++ b/package.json @@ -1,6 +1,6 @@ { "name": "@umbrellio/ucdn", - "version": "0.6.0", + "version": "0.7.0", "bin": "ucdn", "main": "index.js", "repository": "git@github.com:umbrellio/ucdn.git", @@ -14,6 +14,7 @@ "@aws-sdk/client-s3": "^3.421.0", "js-yaml": "^4.1.0", "mime-types": "^2.1.35", + "p-limit": "^6.2.0", "yargs": "^17.7.2" }, "devDependencies": { diff --git a/ucdn b/ucdn index aa120a6..5d9d6a0 100755 --- a/ucdn +++ b/ucdn @@ -59,6 +59,12 @@ const yargsOptions = { type: "string", default: "", }, + concurrencyLimit: { + alias: "processes", + describe: "Limit of concurrent upload processes", + type: "integer", + default: 100, + }, } yargs diff --git a/upload.js b/upload.js index 52026be..d0f485d 100755 --- a/upload.js +++ b/upload.js @@ -2,6 +2,7 @@ const fs = require("fs") const path = require("path") const mime = require("mime-types") const { S3Client, PutObjectCommand } = require("@aws-sdk/client-s3") +const pLimit = require("p-limit") const configDefaults = { region: null, @@ -11,6 +12,7 @@ const configDefaults = { accessKeyId: null, secretAccessKey: null, targetDir: null, + concurrencyLimit: 100, } const getConfig = passedConfig => { @@ -61,10 +63,12 @@ const upload = argv => { return !exclude.includes(ext) }) + const limit = pLimit(config.concurrencyLimit); + const promises = files.map(file => { const key = getKey(file, directory, targetDir) - return uploadFile(s3, config, file, key) - .then(() => console.log("Uploaded", file)) + return limit(uploadFile(s3, config, file, key) + .then(() => console.log("Uploaded", file))) }) return Promise.all(promises) diff --git a/yarn.lock b/yarn.lock index 8e78ece..263f522 100644 --- a/yarn.lock +++ b/yarn.lock @@ -2511,6 +2511,13 @@ p-limit@^3.0.2: dependencies: yocto-queue "^0.1.0" +p-limit@^6.2.0: + version "6.2.0" + resolved "https://registry.yarnpkg.com/p-limit/-/p-limit-6.2.0.tgz#c254d22ba6aeef441a3564c5e6c2f2da59268a0f" + integrity sha512-kuUqqHNUqoIWp/c467RI4X6mmyuojY5jGutNU0wVTmEOOfcuwLqyMVoAi9MKi2Ak+5i9+nhmrK4ufZE8069kHA== + dependencies: + yocto-queue "^1.1.1" + p-locate@^5.0.0: version "5.0.0" resolved "https://registry.yarnpkg.com/p-locate/-/p-locate-5.0.0.tgz#83c8315c6785005e3bd021839411c9e110e6d834" @@ -2968,3 +2975,8 @@ yocto-queue@^0.1.0: version "0.1.0" resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-0.1.0.tgz#0294eb3dee05028d31ee1a5fa2c556a6aaf10a1b" integrity sha512-rVksvsnNCdJ/ohGc6xgPwyN8eheCxsiLM8mxuE/t/mOVqJewPuO1miLpTHQiRgTKCLexL4MeAFVagts7HmNZ2Q== + +yocto-queue@^1.1.1: + version "1.2.1" + resolved "https://registry.yarnpkg.com/yocto-queue/-/yocto-queue-1.2.1.tgz#36d7c4739f775b3cbc28e6136e21aa057adec418" + integrity sha512-AyeEbWOu/TAXdxlV9wmGcR0+yh2j3vYPGOECcIj2S7MkrLyC7ne+oye2BKTItt0ii2PHk4cDy+95+LshzbXnGg==