feat(server): dynamic job concurrency (#2622)

* feat(server): dynamic job concurrency

* styling and add setting info to top of the job list

* regenerate api

* remove DETECT_OBJECT job

---------

Co-authored-by: Alex Tran <alex.tran1502@gmail.com>
This commit is contained in:
Jason Rasmussen
2023-06-01 06:32:51 -04:00
committed by GitHub
parent 656dc08406
commit 2493dfaba3
48 changed files with 1454 additions and 490 deletions

204
server/package-lock.json generated
View File

@@ -10,7 +10,7 @@
"license": "UNLICENSED",
"dependencies": {
"@babel/runtime": "^7.20.13",
"@nestjs/bull": "^0.6.2",
"@nestjs/bullmq": "^1.1.0",
"@nestjs/common": "^9.2.1",
"@nestjs/config": "^2.2.0",
"@nestjs/core": "^9.2.1",
@@ -24,7 +24,7 @@
"archiver": "^5.3.1",
"axios": "^0.26.0",
"bcrypt": "^5.0.1",
"bull": "^4.10.2",
"bullmq": "^3.14.1",
"class-transformer": "^0.5.1",
"class-validator": "^0.14.0",
"cookie-parser": "^1.4.6",
@@ -1507,20 +1507,6 @@
"win32"
]
},
"node_modules/@nestjs/bull": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull/-/bull-0.6.3.tgz",
"integrity": "sha512-CckH9O3t9qSiO4RCzdYvtFSaaMfIhTXMYagV/rtmVvI1SX5XNnxEaQXvtjxDBXF9DB1JE/5AejIl6ICym+MJIw==",
"dependencies": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
},
"peerDependencies": {
"@nestjs/common": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0",
"@nestjs/core": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0",
"bull": "^3.3 || ^4.0.0"
}
},
"node_modules/@nestjs/bull-shared": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-0.1.3.tgz",
@@ -1533,6 +1519,20 @@
"@nestjs/core": "^6.10.11 || ^7.0.0 || ^8.0.0 || ^9.0.0"
}
},
"node_modules/@nestjs/bullmq": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-1.1.0.tgz",
"integrity": "sha512-XloO39ACm9TuB8XOX53iMUaCW5BTQAnZADtX4a9kczJZU/5/cQVBfSCyfzq9L6dfi5EX6w/1Ayyv+5qBQ5yrzw==",
"dependencies": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
},
"peerDependencies": {
"@nestjs/common": "^7.0.0 || ^8.0.0 || ^9.0.0",
"@nestjs/core": "^7.0.0 || ^8.0.0 || ^9.0.0",
"bullmq": "^3.0.0"
}
},
"node_modules/@nestjs/cli": {
"version": "9.4.2",
"resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-9.4.2.tgz",
@@ -4232,30 +4232,56 @@
"node": ">=0.2.0"
}
},
"node_modules/bull": {
"version": "4.10.4",
"resolved": "https://registry.npmjs.org/bull/-/bull-4.10.4.tgz",
"integrity": "sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==",
"node_modules/bullmq": {
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz",
"integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==",
"dependencies": {
"cron-parser": "^4.2.1",
"debuglog": "^1.0.0",
"get-port": "^5.1.1",
"ioredis": "^5.0.0",
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.5.2",
"semver": "^7.3.2",
"uuid": "^8.3.0"
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
}
},
"node_modules/bullmq/node_modules/brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"dependencies": {
"balanced-match": "^1.0.0"
}
},
"node_modules/bullmq/node_modules/glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"dependencies": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
},
"engines": {
"node": ">=12"
},
"funding": {
"url": "https://github.com/sponsors/isaacs"
}
},
"node_modules/bull/node_modules/uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg==",
"bin": {
"uuid": "dist/bin/uuid"
"node_modules/bullmq/node_modules/minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"dependencies": {
"brace-expansion": "^2.0.1"
},
"engines": {
"node": ">=10"
}
},
"node_modules/busboy": {
@@ -5013,14 +5039,6 @@
}
}
},
"node_modules/debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw==",
"engines": {
"node": "*"
}
},
"node_modules/decimal.js": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
@@ -6422,17 +6440,6 @@
"node": ">=8.0.0"
}
},
"node_modules/get-port": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
"integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ==",
"engines": {
"node": ">=8"
},
"funding": {
"url": "https://github.com/sponsors/sindresorhus"
}
},
"node_modules/get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@@ -8429,9 +8436,9 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"node_modules/msgpackr": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.1.tgz",
"integrity": "sha512-jJdrNH8tzfCtT0rjPFryBXjRDQE7rqfLkah4/8B4gYa7NNZYFBcGxqWBtfQpGC+oYyBwlkj3fARk4aooKNPHxg==",
"version": "1.9.2",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.2.tgz",
"integrity": "sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==",
"optionalDependencies": {
"msgpackr-extract": "^3.0.2"
}
@@ -13122,15 +13129,6 @@
"integrity": "sha512-O+6Gs8UeDbyFpbSh2CPEz/UOrrdWPTBYNblZK5CxxLisYt4kGX3Sc+czffFonyjiGSq3jWLwJS/CCJc7tBr4sQ==",
"optional": true
},
"@nestjs/bull": {
"version": "0.6.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull/-/bull-0.6.3.tgz",
"integrity": "sha512-CckH9O3t9qSiO4RCzdYvtFSaaMfIhTXMYagV/rtmVvI1SX5XNnxEaQXvtjxDBXF9DB1JE/5AejIl6ICym+MJIw==",
"requires": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
}
},
"@nestjs/bull-shared": {
"version": "0.1.3",
"resolved": "https://registry.npmjs.org/@nestjs/bull-shared/-/bull-shared-0.1.3.tgz",
@@ -13139,6 +13137,15 @@
"tslib": "2.5.0"
}
},
"@nestjs/bullmq": {
"version": "1.1.0",
"resolved": "https://registry.npmjs.org/@nestjs/bullmq/-/bullmq-1.1.0.tgz",
"integrity": "sha512-XloO39ACm9TuB8XOX53iMUaCW5BTQAnZADtX4a9kczJZU/5/cQVBfSCyfzq9L6dfi5EX6w/1Ayyv+5qBQ5yrzw==",
"requires": {
"@nestjs/bull-shared": "^0.1.3",
"tslib": "2.5.0"
}
},
"@nestjs/cli": {
"version": "9.4.2",
"resolved": "https://registry.npmjs.org/@nestjs/cli/-/cli-9.4.2.tgz",
@@ -15212,25 +15219,48 @@
"resolved": "https://registry.npmjs.org/buffers/-/buffers-0.1.1.tgz",
"integrity": "sha512-9q/rDEGSb/Qsvv2qvzIzdluL5k7AaJOTrw23z9reQthrbF7is4CtlT0DXyO1oei2DCp4uojjzQ7igaSHp1kAEQ=="
},
"bull": {
"version": "4.10.4",
"resolved": "https://registry.npmjs.org/bull/-/bull-4.10.4.tgz",
"integrity": "sha512-o9m/7HjS/Or3vqRd59evBlWCXd9Lp+ALppKseoSKHaykK46SmRjAilX98PgmOz1yeVaurt8D5UtvEt4bUjM3eA==",
"bullmq": {
"version": "3.14.1",
"resolved": "https://registry.npmjs.org/bullmq/-/bullmq-3.14.1.tgz",
"integrity": "sha512-Fom78UKljYsnJmwbROVPx3eFLuVfQjQbw9KCnVupLzT31RQHhFHV2xd/4J4oWl4u34bZ1JmEUfNnqNBz+IOJuA==",
"requires": {
"cron-parser": "^4.2.1",
"debuglog": "^1.0.0",
"get-port": "^5.1.1",
"ioredis": "^5.0.0",
"cron-parser": "^4.6.0",
"glob": "^8.0.3",
"ioredis": "^5.3.2",
"lodash": "^4.17.21",
"msgpackr": "^1.5.2",
"semver": "^7.3.2",
"uuid": "^8.3.0"
"msgpackr": "^1.6.2",
"semver": "^7.3.7",
"tslib": "^2.0.0",
"uuid": "^9.0.0"
},
"dependencies": {
"uuid": {
"version": "8.3.2",
"resolved": "https://registry.npmjs.org/uuid/-/uuid-8.3.2.tgz",
"integrity": "sha512-+NYs2QeMWy+GWFOEm9xnn6HCDp0l7QBD7ml8zLUmJ+93Q5NF0NocErnwkTkXVFNiX3/fpC6afS8Dhb/gz7R7eg=="
"brace-expansion": {
"version": "2.0.1",
"resolved": "https://registry.npmjs.org/brace-expansion/-/brace-expansion-2.0.1.tgz",
"integrity": "sha512-XnAIvQ8eM+kC6aULx6wuQiwVsnzsi9d3WxzV3FpWTGA19F621kwdbsAcFKXgKUHZWsy+mY6iL1sHTxWEFCytDA==",
"requires": {
"balanced-match": "^1.0.0"
}
},
"glob": {
"version": "8.1.0",
"resolved": "https://registry.npmjs.org/glob/-/glob-8.1.0.tgz",
"integrity": "sha512-r8hpEjiQEYlF2QU0df3dS+nxxSIreXQS1qRhMJM0Q5NDdR386C7jb7Hwwod8Fgiuex+k0GFjgft18yvxm5XoCQ==",
"requires": {
"fs.realpath": "^1.0.0",
"inflight": "^1.0.4",
"inherits": "2",
"minimatch": "^5.0.1",
"once": "^1.3.0"
}
},
"minimatch": {
"version": "5.1.6",
"resolved": "https://registry.npmjs.org/minimatch/-/minimatch-5.1.6.tgz",
"integrity": "sha512-lKwV/1brpG6mBUFHtb7NUmtABCb2WZZmm2wNiOA5hAb8VdCS4B3dtMWyvcoViccwAW/COERjXLt0zP1zXUN26g==",
"requires": {
"brace-expansion": "^2.0.1"
}
}
}
},
@@ -15800,11 +15830,6 @@
"ms": "2.1.2"
}
},
"debuglog": {
"version": "1.0.1",
"resolved": "https://registry.npmjs.org/debuglog/-/debuglog-1.0.1.tgz",
"integrity": "sha512-syBZ+rnAK3EgMsH2aYEOLUW7mZSY9Gb+0wUMCFsZvcmiz+HigA0LOcq/HoQqVuGG+EKykunc7QG2bzrponfaSw=="
},
"decimal.js": {
"version": "10.4.3",
"resolved": "https://registry.npmjs.org/decimal.js/-/decimal.js-10.4.3.tgz",
@@ -16867,11 +16892,6 @@
"integrity": "sha512-pjzuKtY64GYfWizNAJ0fr9VqttZkNiK2iS430LtIHzjBEr6bX8Am2zm4sW4Ro5wjWW5cAlRL1qAMTcXbjNAO2Q==",
"dev": true
},
"get-port": {
"version": "5.1.1",
"resolved": "https://registry.npmjs.org/get-port/-/get-port-5.1.1.tgz",
"integrity": "sha512-g/Q1aTSDOxFpchXC4i8ZWvxA1lnPqx/JHqcpIw0/LX9T8x/GBbi6YnlN5nhaKIFkT8oFsscUKgDJYxfwfS6QsQ=="
},
"get-stream": {
"version": "6.0.1",
"resolved": "https://registry.npmjs.org/get-stream/-/get-stream-6.0.1.tgz",
@@ -18386,9 +18406,9 @@
"integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w=="
},
"msgpackr": {
"version": "1.9.1",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.1.tgz",
"integrity": "sha512-jJdrNH8tzfCtT0rjPFryBXjRDQE7rqfLkah4/8B4gYa7NNZYFBcGxqWBtfQpGC+oYyBwlkj3fARk4aooKNPHxg==",
"version": "1.9.2",
"resolved": "https://registry.npmjs.org/msgpackr/-/msgpackr-1.9.2.tgz",
"integrity": "sha512-xtDgI3Xv0AAiZWLRGDchyzBwU6aq0rwJ+W+5Y4CZhEWtkl/hJtFFLc+3JtGTw7nz1yquxs7nL8q/yA2aqpflIQ==",
"requires": {
"msgpackr-extract": "^3.0.2"
}