diff options
| -rw-r--r-- | package-lock.json | 371 | ||||
| -rw-r--r-- | package.json | 4 | ||||
| -rwxr-xr-x | src/jobctl.js | 430 | ||||
| -rwxr-xr-x | src/jobd-master.js | 18 | ||||
| -rwxr-xr-x | src/jobd.js | 69 | ||||
| -rw-r--r-- | src/lib/config.js | 51 | ||||
| -rw-r--r-- | src/lib/logger.js | 20 | ||||
| -rw-r--r-- | src/lib/request-handler.js | 2 | ||||
| -rw-r--r-- | src/lib/server.js | 30 |
9 files changed, 949 insertions, 46 deletions
diff --git a/package-lock.json b/package-lock.json index 95fc36e..9e18e25 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,8 +1,334 @@ { "name": "jobd", "version": "1.7.0", - "lockfileVersion": 1, + "lockfileVersion": 2, "requires": true, + "packages": { + "": { + "version": "1.7.0", + "license": "BSD-2-Clause", + "os": [ + "darwin", + "linux" + ], + "dependencies": { + "columnify": "^1.5.4", + "ini": "^2.0.0", + "lodash": "^4.17.21", + "log4js": "^6.3.0", + "minimist": "^1.2.5", + "mysql": "^2.18.1", + "promise-mysql": "^5.0.2", + "queue": "^6.0.2" + }, + "bin": { + "jobctl": "src/jobctl.js", + "jobd": "src/jobd.js", + "jobd-master": "src/jobd-master.js" + }, + "engines": { + "node": ">=14.0" + } + }, + "node_modules/@types/bluebird": { + "version": "3.5.33", + "resolved": "https://registry.npmjs.org/@types/bluebird/-/bluebird-3.5.33.tgz", + "integrity": "sha512-ndEo1xvnYeHxm7I/5sF6tBvnsA4Tdi3zj1keRKRs12SP+2ye2A27NDJ1B6PqkfMbGAcT+mqQVqbZRIrhfOp5PQ==" + }, + "node_modules/@types/mysql": { + "version": "2.15.17", + "resolved": "https://registry.npmjs.org/@types/mysql/-/mysql-2.15.17.tgz", + "integrity": "sha512-5vlnAFgdjFGqu3fHbd+pp+qL9mMty6c/N65TjsT5H+kfet50Qq4tXWMrD5lm/ftXeiEQwbAndZepB/eaLGaTew==", + "dependencies": { + "@types/node": "*" + } + }, + "node_modules/@types/node": { + "version": "14.14.31", + "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.31.tgz", + "integrity": "sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g==" + }, + "node_modules/ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=", + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/bignumber.js": { + "version": "9.0.0", + "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", + "integrity": "sha512-t/OYhhJ2SD+YGBQcjY8GzzDHEk9f3nerxjtfa6tlMXfe7frs/WozhvCNoGvpM0P3bNf3Gq5ZRMlGr5f3r4/N8A==", + "engines": { + "node": "*" + } + }, + "node_modules/bluebird": { + "version": "3.7.2", + "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", + "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" + }, + "node_modules/clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=", + "engines": { + "node": ">=0.8" + } + }, + "node_modules/columnify": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/columnify/-/columnify-1.5.4.tgz", + "integrity": "sha1-Rzfd8ce2mop8NAVweC6UfuyOeLs=", + "dependencies": { + "strip-ansi": "^3.0.0", + "wcwidth": "^1.0.0" + } + }, + "node_modules/core-util-is": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", + "integrity": "sha1-tf1UIgqivFq1eqtxQMlAdUUDwac=" + }, + "node_modules/date-format": { + "version": "3.0.0", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-3.0.0.tgz", + "integrity": "sha512-eyTcpKOcamdhWJXj56DpQMo1ylSQpcGtGKXcU0Tb97+K56/CF5amAqqqNj0+KvA0iw2ynxtHWFsPDSClCxe48w==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/debug": { + "version": "4.3.1", + "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.1.tgz", + "integrity": "sha512-doEwdvm4PCeK4K3RQN2ZC2BYUBaxwLARCqZmMjtF8a51J2Rb0xpVloFRnCODwqjpwnAoao4pelN8l3RJdv3gRQ==", + "dependencies": { + "ms": "2.1.2" + }, + "engines": { + "node": ">=6.0" + } + }, + "node_modules/defaults": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", + "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", + "dependencies": { + "clone": "^1.0.2" + } + }, + "node_modules/flatted": { + "version": "2.0.2", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", + "integrity": "sha512-r5wGx7YeOwNWNlCA0wQ86zKyDLMQr+/RB8xy74M4hTphfmjlijTSSXGuH8rnvKZnfT9i+75zmd8jcKdMR4O6jA==" + }, + "node_modules/fs-extra": { + "version": "8.1.0", + "resolved": "https://registry.npmjs.org/fs-extra/-/fs-extra-8.1.0.tgz", + "integrity": "sha512-yhlQgA6mnOJUKOsRUFsgJdQCvkKhcz8tlZG5HBQfReYZy46OwLcY+Zia0mtdHsOo9y/hP+CxMN0TU9QxoOtG4g==", + "dependencies": { + "graceful-fs": "^4.2.0", + "jsonfile": "^4.0.0", + "universalify": "^0.1.0" + }, + "engines": { + "node": ">=6 <7 || >=8" + } + }, + "node_modules/graceful-fs": { + "version": "4.2.6", + "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.6.tgz", + "integrity": "sha512-nTnJ528pbqxYanhpDYsi4Rd8MAeaBA67+RZ10CM1m3bTAVFEDcd5AuA4a6W5YkGZ1iNXHzZz8T6TBKLeBuNriQ==" + }, + "node_modules/inherits": { + "version": "2.0.4", + "resolved": "https://registry.npmjs.org/inherits/-/inherits-2.0.4.tgz", + "integrity": "sha512-k/vGaX4/Yla3WzyMCvTQOXYeIHvqOKtnqBduzTHpzpQZzAskKMhZ2K+EnBiSM9zGSoIFeMpXKxa4dYeZIQqewQ==" + }, + "node_modules/ini": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/ini/-/ini-2.0.0.tgz", + "integrity": "sha512-7PnF4oN3CvZF23ADhA5wRaYEQpJ8qygSkbtTXWBeXWXmEVRXK+1ITciHWwHhsjv1TmW0MgacIv6hEi5pX5NQdA==", + "engines": { + "node": ">=10" + } + }, + "node_modules/isarray": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/isarray/-/isarray-1.0.0.tgz", + "integrity": "sha1-u5NdSFgsuhaMBoNJV6VKPgcSTxE=" + }, + "node_modules/jsonfile": { + "version": "4.0.0", + "resolved": "https://registry.npmjs.org/jsonfile/-/jsonfile-4.0.0.tgz", + "integrity": "sha1-h3Gq4HmbZAdrdmQPygWPnBDjPss=", + "dependencies": { + "graceful-fs": "^4.1.6" + } + }, + "node_modules/lodash": { + "version": "4.17.21", + "resolved": "https://registry.npmjs.org/lodash/-/lodash-4.17.21.tgz", + "integrity": "sha512-v2kDEe57lecTulaDIuNTPy3Ry4gLGJ6Z1O3vE1krgXZNrsQ+LFTGHVxVjcXPs17LhbZVGedAJv8XZ1tvj5FvSg==" + }, + "node_modules/log4js": { + "version": "6.3.0", + "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.3.0.tgz", + "integrity": "sha512-Mc8jNuSFImQUIateBFwdOQcmC6Q5maU0VVvdC2R6XMb66/VnT+7WS4D/0EeNMZu1YODmJe5NIn2XftCzEocUgw==", + "dependencies": { + "date-format": "^3.0.0", + "debug": "^4.1.1", + "flatted": "^2.0.1", + "rfdc": "^1.1.4", + "streamroller": "^2.2.4" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/minimist": { + "version": "1.2.5", + "resolved": "https://registry.npmjs.org/minimist/-/minimist-1.2.5.tgz", + "integrity": "sha512-FM9nNUYrRBAELZQT3xeZQ7fmMOBg6nWNmJKTcgsJeaLstP/UODVpGsr5OhXhhXg6f+qtJ8uiZ+PUxkDWcgIXLw==" + }, + "node_modules/ms": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", + "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" + }, + "node_modules/mysql": { + "version": "2.18.1", + "resolved": "https://registry.npmjs.org/mysql/-/mysql-2.18.1.tgz", + "integrity": "sha512-Bca+gk2YWmqp2Uf6k5NFEurwY/0td0cpebAucFpY/3jhrwrVGuxU2uQFCHjU19SJfje0yQvi+rVWdq78hR5lig==", + "dependencies": { + "bignumber.js": "9.0.0", + "readable-stream": "2.3.7", + "safe-buffer": "5.1.2", + "sqlstring": "2.3.1" + }, + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/process-nextick-args": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/process-nextick-args/-/process-nextick-args-2.0.1.tgz", + "integrity": "sha512-3ouUOpQhtgrbOa17J7+uxOTpITYWaGP7/AhoR3+A+/1e9skrzelGi/dXzEYyvbxubEF6Wn2ypscTKiKJFFn1ag==" + }, + "node_modules/promise-mysql": { + "version": "5.0.3", + "resolved": "https://registry.npmjs.org/promise-mysql/-/promise-mysql-5.0.3.tgz", + "integrity": "sha512-qM7ODPO2WUrQqznrDlBUib/+zuQTd1K9qIqX/6gTUiBUDpWu63olSMZG+OSO/PcC9q0aNNhp9qB+ToMWNcoeaw==", + "dependencies": { + "@types/bluebird": "^3.5.26", + "@types/mysql": "^2.15.2", + "bluebird": "^3.5.1", + "mysql": "^2.18.1" + } + }, + "node_modules/queue": { + "version": "6.0.2", + "resolved": "https://registry.npmjs.org/queue/-/queue-6.0.2.tgz", + "integrity": "sha512-iHZWu+q3IdFZFX36ro/lKBkSvfkztY5Y7HMiPlOUjhupPcG2JMfst2KKEpu5XndviX/3UhFbRngUPNKtgvtZiA==", + "dependencies": { + "inherits": "~2.0.3" + } + }, + "node_modules/readable-stream": { + "version": "2.3.7", + "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-2.3.7.tgz", + "integrity": "sha512-Ebho8K4jIbHAxnuxi7o42OrZgF/ZTNcsZj6nRKyUmkhLFq8CHItp/fy6hQZuZmP/n3yZ9VBUbp4zz/mX8hmYPw==", + "dependencies": { + "core-util-is": "~1.0.0", + "inherits": "~2.0.3", + "isarray": "~1.0.0", + "process-nextick-args": "~2.0.0", + "safe-buffer": "~5.1.1", + "string_decoder": "~1.1.1", + "util-deprecate": "~1.0.1" + } + }, + "node_modules/rfdc": { + "version": "1.2.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.2.0.tgz", + "integrity": "sha512-ijLyszTMmUrXvjSooucVQwimGUk84eRcmCuLV8Xghe3UO85mjUtRAHRyoMM6XtyqbECaXuBWx18La3523sXINA==" + }, + "node_modules/safe-buffer": { + "version": "5.1.2", + "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", + "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" + }, + "node_modules/sqlstring": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/sqlstring/-/sqlstring-2.3.1.tgz", + "integrity": "sha1-R1OT/56RR5rqYtyvDKPRSYOn+0A=", + "engines": { + "node": ">= 0.6" + } + }, + "node_modules/streamroller": { + "version": "2.2.4", + "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-2.2.4.tgz", + "integrity": "sha512-OG79qm3AujAM9ImoqgWEY1xG4HX+Lw+yY6qZj9R1K2mhF5bEmQ849wvrb+4vt4jLMLzwXttJlQbOdPOQVRv7DQ==", + "dependencies": { + "date-format": "^2.1.0", + "debug": "^4.1.1", + "fs-extra": "^8.1.0" + }, + "engines": { + "node": ">=8.0" + } + }, + "node_modules/streamroller/node_modules/date-format": { + "version": "2.1.0", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-2.1.0.tgz", + "integrity": "sha512-bYQuGLeFxhkxNOF3rcMtiZxvCBAquGzZm6oWA1oZ0g2THUzivaRhv8uOhdr19LmoobSOLoIAxeUK2RdbM8IFTA==", + "engines": { + "node": ">=4.0" + } + }, + "node_modules/string_decoder": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.1.1.tgz", + "integrity": "sha512-n/ShnvDi6FHbbVfviro+WojiFzv+s8MPMHBczVePfUpDJLwoLT0ht1l4YwBCbi8pJAveEEdnkHyPyTP/mzRfwg==", + "dependencies": { + "safe-buffer": "~5.1.0" + } + }, + "node_modules/strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "dependencies": { + "ansi-regex": "^2.0.0" + }, + "engines": { + "node": ">=0.10.0" + } + }, + "node_modules/universalify": { + "version": "0.1.2", + "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", + "integrity": "sha512-rBJeI5CXAlmy1pV+617WB9J63U6XcazHHF2f2dbJix4XzpUF0RS3Zbj0FGIOCAva5P/d/GBOYaACQ1w+0azUkg==", + "engines": { + "node": ">= 4.0.0" + } + }, + "node_modules/util-deprecate": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", + "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "node_modules/wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=", + "dependencies": { + "defaults": "^1.0.3" + } + } + }, "dependencies": { "@types/bluebird": { "version": "3.5.33", @@ -22,6 +348,11 @@ "resolved": "https://registry.npmjs.org/@types/node/-/node-14.14.31.tgz", "integrity": "sha512-vFHy/ezP5qI0rFgJ7aQnjDXwAMrG0KqqIH7tQG5PPv3BWBayOPIQNBjVc/P6hhdZfMx51REc6tfDNXHUio893g==" }, + "ansi-regex": { + "version": "2.1.1", + "resolved": "https://registry.npmjs.org/ansi-regex/-/ansi-regex-2.1.1.tgz", + "integrity": "sha1-w7M6te42DYbg5ijwRorn7yfWVN8=" + }, "bignumber.js": { "version": "9.0.0", "resolved": "https://registry.npmjs.org/bignumber.js/-/bignumber.js-9.0.0.tgz", @@ -32,6 +363,20 @@ "resolved": "https://registry.npmjs.org/bluebird/-/bluebird-3.7.2.tgz", "integrity": "sha512-XpNj6GDQzdfW+r2Wnn7xiSAd7TM3jzkxGXBGTtWKuSXv1xUV+azxAm8jdWZN06QTQk+2N2XB9jRDkvbmQmcRtg==" }, + "clone": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/clone/-/clone-1.0.4.tgz", + "integrity": "sha1-2jCcwmPfFZlMaIypAheco8fNfH4=" + }, + "columnify": { + "version": "1.5.4", + "resolved": "https://registry.npmjs.org/columnify/-/columnify-1.5.4.tgz", + "integrity": "sha1-Rzfd8ce2mop8NAVweC6UfuyOeLs=", + "requires": { + "strip-ansi": "^3.0.0", + "wcwidth": "^1.0.0" + } + }, "core-util-is": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/core-util-is/-/core-util-is-1.0.2.tgz", @@ -50,6 +395,14 @@ "ms": "2.1.2" } }, + "defaults": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/defaults/-/defaults-1.0.3.tgz", + "integrity": "sha1-xlYFHpgX2f8I7YgUd/P+QBnz730=", + "requires": { + "clone": "^1.0.2" + } + }, "flatted": { "version": "2.0.2", "resolved": "https://registry.npmjs.org/flatted/-/flatted-2.0.2.tgz", @@ -209,6 +562,14 @@ "safe-buffer": "~5.1.0" } }, + "strip-ansi": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/strip-ansi/-/strip-ansi-3.0.1.tgz", + "integrity": "sha1-ajhfuIU9lS1f8F0Oiq+UJ43GPc8=", + "requires": { + "ansi-regex": "^2.0.0" + } + }, "universalify": { "version": "0.1.2", "resolved": "https://registry.npmjs.org/universalify/-/universalify-0.1.2.tgz", @@ -218,6 +579,14 @@ "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", "integrity": "sha1-RQ1Nyfpw3nMnYvvS1KKJgUGaDM8=" + }, + "wcwidth": { + "version": "1.0.1", + "resolved": "https://registry.npmjs.org/wcwidth/-/wcwidth-1.0.1.tgz", + "integrity": "sha1-8LDc+RW8X/FSivrbLA4XtTLaL+g=", + "requires": { + "defaults": "^1.0.3" + } } } } diff --git a/package.json b/package.json index 50e1726..89e46c5 100644 --- a/package.json +++ b/package.json @@ -13,7 +13,8 @@ }, "bin": { "jobd": "src/jobd.js", - "jobd-master": "src/jobd-master.js" + "jobd-master": "src/jobd-master.js", + "jobctl": "src/jobctl.js" }, "keywords": [], "author": "Evgeny Zinoviev", @@ -26,6 +27,7 @@ "node": ">=14.0" }, "dependencies": { + "columnify": "^1.5.4", "ini": "^2.0.0", "lodash": "^4.17.21", "log4js": "^6.3.0", diff --git a/src/jobctl.js b/src/jobctl.js new file mode 100755 index 0000000..2a1fa30 --- /dev/null +++ b/src/jobctl.js @@ -0,0 +1,430 @@ +#!/usr/bin/env node +const minimist = require('minimist') +const loggerModule = require('./lib/logger') +const config = require('./lib/config') +const package_json = require('../package.json') +const os = require('os') +const path = require('path') +const fs = require('fs/promises') +const {Connection, RequestMessage} = require('./lib/server') +const {isNumeric} = require('./lib/util') +const columnify = require('columnify') + +const DEFAULT_CONFIG_PATH = path.join(os.homedir(), '.jobctl.conf') + +const WORKER_COMMANDS = { + 'list-targets': workerListTargets, + 'memory-usage': workerMemoryUsage, + 'poll': workerPoll, + 'set-target-concurrency': workerSetTargetConcurrency, + 'pause': workerPause, + 'continue': workerContinue +} + +const MASTER_COMMANDS = { + 'list-workers': masterListWorkers, + // 'list-workers-memory-usage': masterListWorkersMemoryUsage, + 'memory-usage': masterMemoryUsage, + 'poke': masterPoke, + + // we can just reuse worker functions here, as they do the same + 'pause': workerPause, + 'continue': workerContinue, +} + +/** + * @type {Logger} + */ +let logger + +/** + * @type {Connection} + */ +let connection + + +main().catch(e => { + console.error(e) + process.exit(1) +}) + + +async function main() { + const argv = await initApp('jobctl') + if (!argv.length) + usage() + + const isMaster = config.get('master') + + logger.info('Working mode: ' + (isMaster ? 'master' : 'worker')) + logger.trace('Command arguments: ', argv) + + let availableCommands = isMaster ? MASTER_COMMANDS : WORKER_COMMANDS + let command = argv.shift() + if (!(command in availableCommands)) { + logger.error(`Unsupported command: '${command}'`) + process.exit(1) + } + + let host = config.get('host') + let port = config.get('port') + + // connect to instance + try { + connection = new Connection() + await connection.connect(host, port) + + logger.info('Successfully connected.') + } catch (error) { + logger.error('Connection failure:', error) + process.exit(1) + } + + try { + await availableCommands[command](argv) + } catch (e) { + logger.error(e.message) + } + + connection.close() + + // initWorker() + // initRequestHandler() + // initServer() + // connectToMaster() +} + +async function initApp(appName) { + if (process.argv.length < 3) + usage() + + process.on('SIGINT', term) + process.on('SIGTERM', term) + + const argv = minimist(process.argv.slice(2), { + boolean: ['master', 'version', 'help'], + string: ['host', 'port', 'config', 'log-level'], + stopEarly: true, + default: { + config: DEFAULT_CONFIG_PATH + } + }) + + if (argv.help) + usage() + + if (argv.version) { + console.log(package_json.version) + process.exit(0) + } + + // read config + if (await exists(argv.config)) { + try { + config.parseJobctlConfig(argv.config, { + master: argv.master, + log_level: argv['log-level'], + host: argv.host, + port: parseInt(argv.port, 10), + }) + } catch (e) { + console.error(`config parsing error: ${e.message}`) + process.exit(1) + } + } + + // init logger + await loggerModule.init({ + levelConsole: config.get('log_level'), + disableTimestamps: true + }) + logger = loggerModule.getLogger(appName) + + process.title = appName + + /// /// /// + /// \\\ \\\ + /// /// /// + /// \\\ \\\ + /// /// /// + /* * * * * */ + /* */ + /* ^_^ */ + /* */ + /* '_' */ + /* */ + /* <_< */ + /* */ + /* >_> */ + /* */ + /* * * * * */ + /// /// /// + /// \\\ \\\ + /// /// /// + /// \\\ \\\ + /// /// /// + + return argv['_'] || [] +} + +async function workerListTargets() { + try { + let response = await connection.sendRequest(new RequestMessage('status')) + const rows = [] + const columns = [ + 'target', + 'concurrency', + 'length', + 'paused' + ] + for (const target in response.data.targets) { + const row = [ + target, + response.data.targets[target].concurrency, + response.data.targets[target].length, + response.data.targets[target].paused ? 'yes' : 'no' + ] + rows.push(row) + } + + table(columns, rows) + } catch (error) { + logger.error(error.message) + logger.trace(error) + } +} + +async function workerMemoryUsage() { + try { + let response = await connection.sendRequest(new RequestMessage('status')) + const columns = ['what', 'value'] + const rows = [] + for (const what in response.data.memoryUsage) + rows.push([what, response.data.memoryUsage[what]]) + rows.push(['pendingJobPromises', response.data.jobPromisesCount]) + table(columns, rows) + } catch (error) { + logger.error(error.message) + logger.trace(error) + } +} + +async function workerPoll(argv) { + return await sendCommandForTargets(argv, 'poll') +} + +async function workerPause(argv) { + return await sendCommandForTargets(argv, 'pause') +} + +async function workerContinue(argv) { + return await sendCommandForTargets(argv, 'continue') +} + +async function workerSetTargetConcurrency(argv) { + if (argv.length !== 2) + throw new Error('Invalid number of arguments.') + + let [target, concurrency] = argv + if (!isNumeric(concurrency)) + throw new Error(`'concurrency' must be a number.`) + + concurrency = parseInt(concurrency, 10) + + try { + let response = await connection.sendRequest( + new RequestMessage('set-target-concurrency', { + target, concurrency + }) + ) + + if (response.error) + throw new Error(`Worker error: ${response.error}`) + + console.log(response.data) + } catch (error) { + logger.error(error.message) + logger.trace(error) + } +} + +async function masterPoke(argv) { + return await sendCommandForTargets(argv, 'poke') +} + +async function masterMemoryUsage() { + try { + let response = await connection.sendRequest(new RequestMessage('status')) + const columns = ['what', 'value'] + const rows = [] + for (const what in response.data.memoryUsage) + rows.push([what, response.data.memoryUsage[what]]) + table(columns, rows) + } catch (error) { + logger.error(error.message) + logger.trace(error) + } +} + +async function masterListWorkers() { + try { + let response = await connection.sendRequest(new RequestMessage('status', {poll_workers: true})) + const columns = ['worker', 'targets', 'concurrency', 'length', 'paused'] + const rows = [] + for (const worker of response.data.workers) { + let remoteAddr = `${worker.remoteAddr}:${worker.remotePort}` + let targets = Object.keys(worker.workerStatus.targets) + let concurrencies = targets.map(t => worker.workerStatus.targets[t].concurrency) + let lengths = targets.map(t => worker.workerStatus.targets[t].length) + let pauses = targets.map(t => worker.workerStatus.targets[t].paused ? 'yes' : 'no') + rows.push([ + remoteAddr, + targets.join("\n"), + concurrencies.join("\n"), + lengths.join("\n"), + pauses.join("\n") + ]) + } + table(columns, rows) + } catch (error) { + logger.error(error.message) + logger.trace(error) + } +} + +async function sendCommandForTargets(targets, command) { + if (!targets.length) + throw new Error('No targets specified.') + + try { + let response = await connection.sendRequest( + new RequestMessage(command, {targets}) + ) + + if (response.error) + throw new Error(`Worker error: ${response.error}`) + + console.log(response.data) + } catch (error) { + logger.error(error.message) + logger.trace(error) + } +} + + +function usage(exitCode = 0) { + let s = `${process.argv[1]} [OPTIONS] COMMAND + +Worker commands: + list-targets Print list of targets, their length and inner state. + memory-usage Print info about memory usage of the worker. + poll <...TARGETS> Ask worker to get tasks for specified targets. + + Example: + $ jobctl poke t1 t2 t3 + + set-target-concurrency <target> <concurrency> + Set concurrency of the target. + + pause <...TARGETS> Pause specified or all targets. + continue <...TARGETS> Pause specified or all targets. + +Master commands: + list-workers Print list of connected workers and their state. + memory-usage Print info about memory usage. + poke <...TARGETS> Poke specified targets. + pause <...TARGETS> Send pause() to all workers serving specified targets. + If no targets specified, just sends pause() to all + connected workers. + continue <...TARGETS> Send continue() to all workers serving specified + targets. If no targets specified, just sends pause() + to all connected workers. + +Options: + --master Connect to jobd-master instance. + --host Address of jobd or jobd-master instance. + --port Port. Default: 7080 when --master is not used, + 7081 otherwise. + --config <path> Path to config. Default: ~/.jobctl.conf + Required for connecting to password-protected + instances. + --log-level <level> 'error', 'warn', 'info', 'debug' or 'trace'. + Default: warn + --help: Show this help. + --version: Print version. + +Configuration file + Config file is required for connecting to password-protected jobd instances. + It can also be used to store hostname, port and log level. + + Here's an example of possible ~/.jobctl.conf file: + + ;password = + hostname = 1.2.3.4 + port = 7080 + log_level = warn + master = true +` + + console.log(s) + process.exit(exitCode) +} + +function term() { + if (logger) + logger.info('shutdown') + + loggerModule.shutdown(function() { + process.exit() + }) +} + +async function exists(file) { + let exists + try { + await fs.stat(file) + exists = true + } catch (error) { + exists = false + } + return exists +} + +function table(columns, rows) { + const maxColumnSize = {} + for (const c of columns) + maxColumnSize[c] = c.length + + rows = rows.map(values => { + if (!Array.isArray(values)) + throw new Error('row must be array, got', values) + + let row = {} + for (let i = 0; i < columns.length; i++) { + let value = String(values[i]) + row[columns[i]] = value + + let width + if (value.indexOf('\n') !== -1) { + width = Math.max(...value.split('\n').map(s => s.length)) + } else { + width = value.length + } + + if (width > maxColumnSize[columns[i]]) + maxColumnSize[columns[i]] = width + } + + return row + }) + + console.log(columnify(rows, { + columns, + preserveNewLines: true, + columnSplitter: ' | ', + headingTransform: (text) => { + const repeat = () => '-'.repeat(maxColumnSize[text]) + return `${text.toUpperCase()}\n${repeat()}` + } + })) +}
\ No newline at end of file diff --git a/src/jobd-master.js b/src/jobd-master.js index 5839b1a..c926dc0 100755 --- a/src/jobd-master.js +++ b/src/jobd-master.js @@ -8,6 +8,8 @@ const {validateObjectSchema, validateTargetsListFormat} = require('./lib/data-va const RequestHandler = require('./lib/request-handler') const package_json = require('../package.json') +const DEFAULT_CONFIG_PATH = "/etc/jobd-master.conf" + /** * @type {Logger} */ @@ -51,7 +53,12 @@ async function initApp(appName) { process.on('SIGINT', term) process.on('SIGTERM', term) - const argv = minimist(process.argv.slice(2)) + const argv = minimist(process.argv.slice(2), { + boolean: ['help', 'version'], + default: { + config: DEFAULT_CONFIG_PATH + } + }) if (argv.help) { usage() @@ -63,9 +70,6 @@ async function initApp(appName) { process.exit(0) } - if (!argv.config) - throw new Error('--config option is required') - // read config try { config.parseMasterConfig(argv.config) @@ -295,9 +299,9 @@ function usage() { let s = `${process.argv[1]} OPTIONS Options: - --config <path> - --help - --version` + --config <path> Path to config. Default: ${DEFAULT_CONFIG_PATH} + --help Show this help. + --version Print version.` console.log(s) } diff --git a/src/jobd.js b/src/jobd.js index de8807f..0d8af32 100755 --- a/src/jobd.js +++ b/src/jobd.js @@ -17,11 +17,12 @@ const { Worker, STATUS_MANUAL, JOB_NOTFOUND, - JOB_ACCEPTED, JOB_IGNORED } = require('./lib/worker') const package_json = require('../package.json') +const DEFAULT_CONFIG_PATH = "/etc/jobd.conf" + /** * @type {Worker} */ @@ -72,7 +73,12 @@ async function initApp(appName) { process.on('SIGINT', term) process.on('SIGTERM', term) - const argv = minimist(process.argv.slice(2)) + const argv = minimist(process.argv.slice(2), { + boolean: ['help', 'version'], + default: { + config: DEFAULT_CONFIG_PATH + } + }) if (argv.help) { usage() @@ -84,9 +90,6 @@ async function initApp(appName) { process.exit(0) } - if (!argv.config) - throw new Error('--config option is required') - // read config try { config.parseWorkerConfig(argv.config) @@ -337,6 +340,9 @@ function onSetTargetConcurrency(data, requestNo, connection) { ['concurrency', 'i', true], ['target', 's', true], ]) + + if (data.concurrency <= 0) + throw new Error('Invalid concurrency value.') } catch (e) { connection.send( new ResponseMessage(requestNo) @@ -395,42 +401,49 @@ function connectToMaster() { return } - const connection = new Connection() - connection.connect(host, port) + async function connect() { + const connection = new Connection() + await connection.connect(host, port) - connection.on('connect', function() { - connection.sendRequest( - new RequestMessage('register-worker', { - targets: worker.getTargets() - }) - ) - .then(response => { + try { + let response = await connection.sendRequest( + new RequestMessage('register-worker', { + targets: worker.getTargets() + }) + ) logger.debug('connectToMaster: response:', response) - }) - .catch(error => { + } catch (error) { logger.error('connectToMaster: error while awaiting response:', error) + } + + connection.on('close', () => { + logger.warn(`connectToMaster: connection closed`) + tryToConnect() }) - }) - connection.on('close', () => { - logger.warn(`connectToMaster: connection closed`) + connection.on('request-message', (message, connection) => { + requestHandler.process(message, connection) + }) + } + + function tryToConnect(now = false) { setTimeout(() => { - connectToMaster() - }, config.get('master_reconnect_timeout') * 1000) - }) + connect().catch(error => { + logger.warn(`connectToMaster: connection failed`, error) + }) + }, now ? 0 : config.get('master_reconnect_timeout') * 1000) + } - connection.on('request-message', (message, connection) => { - requestHandler.process(message, connection) - }) + tryToConnect(true) } function usage() { let s = `${process.argv[1]} OPTIONS Options: - --config <path> - --help - --version` + --config <path> Path to config. Default: ${DEFAULT_CONFIG_PATH} + --help Show this help. + --version Print version.` console.log(s) } diff --git a/src/lib/config.js b/src/lib/config.js index ac711fd..73a6226 100644 --- a/src/lib/config.js +++ b/src/lib/config.js @@ -38,7 +38,15 @@ function processScheme(source, scheme) { case 'object': if (typeof value !== 'object') throw new Error(`'${key}' must be an object`) + break + case 'boolean': + if (value !== null) { + value = value.trim() + value = ['true', '1'].includes(value) + } else { + value = false + } break } @@ -115,6 +123,40 @@ function parseMasterConfig(file) { } /** + * @param {string} file + * @param {{ + * master: boolean, + * log_level: string|undefined, + * host: string, + * port: int, + * }} inputOptions + */ +function parseJobctlConfig(file, inputOptions) { + config = {} + const raw = readFile(file) + + Object.assign(config, processScheme(raw, { + master: {type: 'boolean'}, + password: {}, + log_level: {default: 'warn'}, + })) + + if (inputOptions.master) + config.master = inputOptions.master + Object.assign(config, processScheme(raw, { + host: {default: '127.0.0.1'}, + port: {default: config.master ? 7081 : 7080, type: 'int'} + })) + + for (let key of ['log_level', 'host', 'port']) { + if (inputOptions[key]) + config[key] = inputOptions[key] + } + + // console.log('parseJobctlConfig [2]', config) +} + +/** * @param {string|null} key * @return {string|number|object} */ @@ -131,8 +173,17 @@ function get(key = null) { return config[key] } +/** + * @param {object} opts + */ +// function set(opts) { +// Object.assign(config, opts) +// } + module.exports = { parseWorkerConfig, parseMasterConfig, + parseJobctlConfig, get, + // set, }
\ No newline at end of file diff --git a/src/lib/logger.js b/src/lib/logger.js index 54c9d54..8a44e07 100644 --- a/src/lib/logger.js +++ b/src/lib/logger.js @@ -3,13 +3,16 @@ const fs = require('fs/promises') const fsConstants = require('fs').constants const util = require('./util') +const ALLOWED_LEVELS = ['trace', 'debug', 'info', 'warn', 'error'] + module.exports = { /** * @param {string} file * @param {string} levelFile * @param {string} levelConsole + * @param {boolean} disableTimestamps */ - async init({file, levelFile, levelConsole}) { + async init({file, levelFile, levelConsole, disableTimestamps=false}) { const categories = { default: { appenders: ['stdout-filter'], @@ -17,19 +20,30 @@ module.exports = { } } + if (!ALLOWED_LEVELS.includes(levelConsole)) + throw new Error(`Level ${levelConsole} is not allowed.`) + const appenders = { stdout: { type: 'stdout', - level: 'trace' + level: 'trace', }, 'stdout-filter': { type: 'logLevelFilter', appender: 'stdout', - level: levelConsole + level: levelConsole, } } + if (disableTimestamps) + appenders.stdout.layout = { + type: 'pattern', + pattern: '%[%p [%c]%] %m', + } if (file) { + if (!ALLOWED_LEVELS.includes(levelFile)) + throw new Error(`Level ${levelFile} is not allowed.`) + let exists try { await fs.stat(file) diff --git a/src/lib/request-handler.js b/src/lib/request-handler.js index 4ab06fb..4330b6b 100644 --- a/src/lib/request-handler.js +++ b/src/lib/request-handler.js @@ -1,5 +1,5 @@ const {getLogger} = require('./logger') -const {ResponseMessage} = require('./server') +const {ResponseMessage, Connection} = require('./server') class RequestHandler { diff --git a/src/lib/server.js b/src/lib/server.js index 618ca8c..051b8be 100644 --- a/src/lib/server.js +++ b/src/lib/server.js @@ -285,12 +285,19 @@ class Connection extends EventEmitter { */ this._requestPromises = {} + /** + * @type {Promise} + * @private + */ + this._connectPromise = null + this._setLogger() } /** * @param {string} host * @param {number} port + * @return {Promise} */ connect(host, port) { if (this.socket !== null) @@ -298,14 +305,18 @@ class Connection extends EventEmitter { this._isOutgoing = true + this.logger.trace(`Connecting to ${host}:${port}`) + this.socket = new net.Socket() - this.socket.connect({host, port}) + this.socket.connect(port, host) this.remoteAddress = host this.remotePort = port this._setLogger() this._setSocketEvents() + + return this._connectPromise = createCallablePromise() } /** @@ -616,14 +627,19 @@ class Connection extends EventEmitter { } for (const no in this._requestPromises) { - this._requestPromises[no].reject(new Error('socket is closed')) + this._requestPromises[no].reject(new Error('Socket is closed')) } this._requestPromises = {} } onConnect = () => { - this.logger.debug('connection established') + if (this._connectPromise) { + this._connectPromise.resolve() + this._connectPromise = null + } + + this.logger.debug('Connection established.') this.emit('connect') } @@ -642,12 +658,16 @@ class Connection extends EventEmitter { onClose = (hadError) => { this._handleClose() - this.logger.debug(`socket closed` + (hadError ? ` with error` : '')) + this.logger.debug(`Socket closed` + (hadError ? ` with error` : '')) } onError = (error) => { + if (this._connectPromise) { + this._connectPromise.reject(error) + this._connectPromise = null + } this._handleClose() - this.logger.warn(`socket error:`, error) + this.logger.warn(`Socket error:`, error) } } |