diff options
-rw-r--r-- | app.js (renamed from index.js) | 53 | ||||
-rw-r--r-- | cookies-storage.js | 50 | ||||
-rw-r--r-- | lib/browser.js (renamed from browser.js) | 52 | ||||
-rw-r--r-- | lib/cookies-storage.js | 80 | ||||
-rw-r--r-- | lib/logging.js | 35 | ||||
-rw-r--r-- | package-lock.json | 435 | ||||
-rw-r--r-- | package.json | 7 | ||||
-rw-r--r-- | systemd/rt-pupflare@.service | 2 |
8 files changed, 622 insertions, 92 deletions
@@ -1,12 +1,13 @@ -const cookiesStorage = require('./cookies-storage') -const browser = require('./browser') +const cookiesStorage = require('./lib/cookies-storage') +const browser = require('./lib/browser') const {singlePageWrapper, PageWrapper} = browser const os = require('os') const path = require('path') +const logging = require('./lib/logging') const argv = require('minimist')(process.argv.slice(2), { string: ['retries', 'timeout', 'cookies', 'port', 'proxy'], - boolean: ['no-sandbox', 'headful', 'reuse'], + boolean: ['no-sandbox', 'headful', 'reuse', 'verbose'], stopEarly: true, default: { port: 3000, @@ -14,16 +15,23 @@ const argv = require('minimist')(process.argv.slice(2), { timeout: 30000, cookies: path.join(os.homedir(), '.rt-pupflare-cookies.json'), reuse: false, + verbose: false, } }) +let logger = null let reusePage = argv.reuse + const maxTryCount = parseInt(argv.retries) const loadingTimeout = parseInt(argv.timeout) const Koa = require('koa'); -const Router = require('@koa/router'); +const bodyParser = require('koa-bodyparser') +const Router = require('@koa/router') + const app = new Koa(); +app.use(bodyParser()) + const router = new Router(); @@ -50,22 +58,22 @@ async function requestHandler(ctx, next) { }) resolve() } - - pageWrapper = reusePage ? singlePageWrapper : new PageWrapper() - const page = await pageWrapper.getPage(fInterceptionNeeded, fInterception) - - // not tested - if (ctx.method === "POST") { - await page.removeAllListeners('request') - await page.setRequestInterception(true) - page.on('request', interceptedRequest => { - interceptedRequest.continue({ + const fPostInterception = (request) => { + if (ctx.method === 'POST') { + return { 'method': 'POST', - 'postData': ctx.request.rawBody - }) - }) + 'postData': ctx.request.rawBody, + 'headers': { + ...request.headers, + 'Content-Type': 'application/x-www-form-urlencoded', + } + } + } } + pageWrapper = reusePage ? singlePageWrapper : new PageWrapper() + const page = await pageWrapper.getPage(fInterceptionNeeded, fInterception, fPostInterception) + try { let tryCount = 0 let response = await page.goto(ctx.query.url, { @@ -85,7 +93,7 @@ async function requestHandler(ctx, next) { tryCount++; } - myResult.data = await response.text() + myResult.data = (await response.buffer()).toString('base64') myResult.headers = await response.headers() resolve() @@ -119,6 +127,11 @@ router (async () => { + process.title = 'rt-pupflare' + + logging.configure(argv.verbose) + logger = logging.getLogger('app') + cookiesStorage.setFileName(argv.cookies) await browser.launch({ @@ -131,8 +144,8 @@ router .use(router.allowedMethods()) app.on('error', (error) => { - console.error('[app.onerror]', error) + logger.error('app error:', error) }) app.listen(parseInt(argv.port), '127.0.0.1') -})(); +})()
\ No newline at end of file diff --git a/cookies-storage.js b/cookies-storage.js deleted file mode 100644 index 8c9231b..0000000 --- a/cookies-storage.js +++ /dev/null @@ -1,50 +0,0 @@ -const fs = require('fs').promises - -let storageFileName = null - -async function exists(path) { - try { - await fs.access(path) - return true - } catch { - return false - } -} - -module.exports = { - async save(newCookies) { - let currentCookies = await this.get() - if (currentCookies.length) { - for (let newCookie of newCookies) { - if (!currentCookies.length) - break - let i = currentCookies.findIndex((c) => { - return c.domain === newCookie.domain - && c.path === newCookie.path - && c.name === newCookie.name - }) - if (i !== -1) - currentCookies.splice(i, 1) - } - } - currentCookies.push(...newCookies) - // console.log('[cookies.save] saving cookies:', currentCookies) - await fs.writeFile(storageFileName, JSON.stringify(currentCookies, null, 2), 'utf-8') - }, - - async get() { - if (!(await exists(storageFileName))) - return [] - - try { - return JSON.parse(await fs.readFile(storageFileName, 'utf-8')) - } catch (e) { - console.error(e) - return [] - } - }, - - setFileName(name) { - storageFileName = name - } -};
\ No newline at end of file diff --git a/browser.js b/lib/browser.js index d1f5c50..d0f7861 100644 --- a/browser.js +++ b/lib/browser.js @@ -1,7 +1,11 @@ const puppeteer = require("puppeteer-extra"); const StealthPlugin = require('puppeteer-extra-plugin-stealth'); const cookiesStorage = require("./cookies-storage"); -puppeteer.use(StealthPlugin()); +puppeteer.use(StealthPlugin()) + +const {getLogger} = require('./logging') + +const logger = getLogger('browser') const chromeOptions = { @@ -10,19 +14,21 @@ const chromeOptions = { }; let browser = null -let cdpClient = null class PageWrapper { constructor() { this.intrNeededCallback = null this.intrCallback = null + this.intrPostCallback = null this.page = null + this.cdpClient = null } - async getPage(interceptionNeededCallback, interceptCallback) { + async getPage(interceptionNeededCallback, interceptCallback, postInterceptCallback) { this.intrCallback = interceptCallback this.intrNeededCallback = interceptionNeededCallback + this.intrPostCallback = postInterceptCallback if (this.page !== null && this.page.isClosed()) { this.page.removeAllListeners && this.page.removeAllListeners() @@ -33,38 +39,50 @@ class PageWrapper { return this.page this.page = await browser.newPage() + + let cookies = await cookiesStorage.get() + // logger.debug('loaded cookies:', cookies) + await this.page.setCookie(...cookies) + this.page.on('domcontentloaded', async () => { try { let cookies = await this.page.cookies(); if (cookies) await cookiesStorage.save(cookies) } catch (e) { - console.warn(e) + logger.error('page.cookies() failed:', e) } }) - await this.page.setCookie(...(await cookiesStorage.get())) + await this.page.removeAllListeners('request') + await this.page.setRequestInterception(true) + this.page.on('request', async request => { + let contData = this.intrPostCallback(request) + await request.continue(contData) + }) - cdpClient = await this.page.target().createCDPSession(); - await cdpClient.send('Network.setRequestInterception', { - patterns: [{ - urlPattern: '*', - resourceType: 'Document', - interceptionStage: 'HeadersReceived' - }], + this.cdpClient = await this.page.target().createCDPSession(); + await this.cdpClient.send('Network.enable') + await this.cdpClient.send('Network.setRequestInterception', { + patterns: [ + { + urlPattern: '*', + resourceType: 'Document', + interceptionStage: 'HeadersReceived' + } + ], }) - await cdpClient.on('Network.requestIntercepted', async e => { - let obj = { interceptionId: e.interceptionId } + await this.cdpClient.on('Network.requestIntercepted', async e => { + let obj = {interceptionId: e.interceptionId} if (this.intrNeededCallback && this.intrNeededCallback(e) === true) { - let ret = await cdpClient.send('Network.getResponseBodyForInterception', { + let ret = await this.cdpClient.send('Network.getResponseBodyForInterception', { interceptionId: e.interceptionId }) this.intrCallback(ret, e.responseHeaders) obj['errorReason'] = 'BlockedByClient' } - await cdpClient.send('Network.continueInterceptedRequest', obj) + await this.cdpClient.send('Network.continueInterceptedRequest', obj) }) - return this.page } } diff --git a/lib/cookies-storage.js b/lib/cookies-storage.js new file mode 100644 index 0000000..15714ca --- /dev/null +++ b/lib/cookies-storage.js @@ -0,0 +1,80 @@ +const fs = require('fs').promises +const {Mutex} = require('async-mutex') +const {getLogger} = require('./logging') + +const logger = getLogger('CookiesStorage') + +let storageFileName = null +const mutex = new Mutex() + +async function exists(path) { + try { + await fs.access(path) + return true + } catch { + return false + } +} + +function cookiesAsHashed(cookies) { + if (!cookies.length) + return {} + + const map = {} + const sep = '|;|;' + for (const c of cookies) { + const k = `${c.domain}${sep}${c.path}${sep}${c.name}` + map[k] = c + } + return map +} + +module.exports = { + async save(newCookies) { + let currentCookies = await this.get() + + await mutex.runExclusive(async () => { + if (currentCookies.length) { + for (let newCookie of newCookies) { + if (!currentCookies.length) + break + let i = currentCookies.findIndex((oldCookie) => { + return oldCookie.domain === newCookie.domain + && oldCookie.path === newCookie.path + && oldCookie.name === newCookie.name + && ( + oldCookie.value !== newCookie.value + || oldCookie.expires !== newCookie.expires + ) + }) + if (i !== -1) { + let removed = currentCookies.splice(i, 1) + // logger.debug('removed cookie:', removed, 'instead got new one:', newCookie) + } + } + } + + const cookiesMap = Object.assign({}, cookiesAsHashed(currentCookies), cookiesAsHashed(newCookies)) + await fs.writeFile(storageFileName, JSON.stringify(Object.values(cookiesMap), null, 2), 'utf-8') + }) + }, + + async get() { + if (!(await exists(storageFileName))) + return [] + + try { + const raw = await mutex.runExclusive(async () => { + return await fs.readFile(storageFileName, 'utf-8') + }) + return JSON.parse(raw) + } catch (e) { + logger.error('Failed to parse storage:', e) + return [] + } + }, + + setFileName(name) { + storageFileName = name + } +};
\ No newline at end of file diff --git a/lib/logging.js b/lib/logging.js new file mode 100644 index 0000000..bca6eb6 --- /dev/null +++ b/lib/logging.js @@ -0,0 +1,35 @@ +const log4js = require('log4js') + +module.exports = { + configure(verbose) { + const categories = { + default: { + appenders: ['stdout-filter'], + level: 'trace' + } + } + const appenders = { + stdout: { + type: 'stdout', + level: 'warn' + }, + 'stdout-filter': { + type: 'logLevelFilter', + appender: 'stdout', + level: verbose ? 'debug' : 'warn' + } + } + log4js.configure({ + appenders, + categories + }) + }, + + getLogger(...args) { + return log4js.getLogger(...args) + }, + + shutdown(cb) { + log4js.shutdown(cb) + } +} diff --git a/package-lock.json b/package-lock.json index b0cf47f..5304f9b 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,16 +1,19 @@ { - "name": "pupflare", + "name": "rt-pupflare", "version": "1.0.0", "lockfileVersion": 2, "requires": true, "packages": { "": { - "name": "pupflare", + "name": "rt-pupflare", "version": "1.0.0", "license": "ISC", "dependencies": { "@koa/router": "^10.1.1", + "async-mutex": "^0.3.2", "koa": "^2.13.4", + "koa-bodyparser": "^4.3.0", + "log4js": "^6.4.2", "minimist": "^1.2.5", "puppeteer": "^13.4.1", "puppeteer-extra": "^3.2.3", @@ -250,6 +253,14 @@ "node": ">=0.10.0" } }, + "node_modules/async-mutex": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.2.tgz", + "integrity": "sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==", + "dependencies": { + "tslib": "^2.3.1" + } + }, "node_modules/balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -324,6 +335,14 @@ "node": "*" } }, + "node_modules/bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/cache-content-type": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz", @@ -336,6 +355,18 @@ "node": ">= 6.0.0" } }, + "node_modules/call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "dependencies": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -365,6 +396,17 @@ "node": ">= 0.12.0" } }, + "node_modules/co-body": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/co-body/-/co-body-6.1.0.tgz", + "integrity": "sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==", + "dependencies": { + "inflation": "^2.0.0", + "qs": "^6.5.2", + "raw-body": "^2.3.3", + "type-is": "^1.6.16" + } + }, "node_modules/concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -401,6 +443,11 @@ "node": ">= 0.8" } }, + "node_modules/copy-to": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/copy-to/-/copy-to-2.0.1.tgz", + "integrity": "sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU=" + }, "node_modules/cross-fetch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", @@ -409,6 +456,14 @@ "node-fetch": "2.6.7" } }, + "node_modules/date-format": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-4.0.5.tgz", + "integrity": "sha512-zBhRiN/M0gDxUoM2xRtzTjJzSg0XEi1ofYpF84PfXeS3hN2PsGxmc7jw3DNQtFlimRbMmob5FC3G0cJq6jQQpw==", + "engines": { + "node": ">=4.0" + } + }, "node_modules/debug": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", @@ -530,6 +585,11 @@ "node": ">=8" } }, + "node_modules/flatted": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", + "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==" + }, "node_modules/for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -580,6 +640,24 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, + "node_modules/function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "node_modules/get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "dependencies": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", @@ -618,6 +696,17 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" }, + "node_modules/has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "dependencies": { + "function-bind": "^1.1.1" + }, + "engines": { + "node": ">= 0.4.0" + } + }, "node_modules/has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", @@ -690,6 +779,17 @@ "node": ">= 6" } }, + "node_modules/iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "dependencies": { + "safer-buffer": ">= 2.1.2 < 3" + }, + "engines": { + "node": ">=0.10.0" + } + }, "node_modules/ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", @@ -709,6 +809,14 @@ } ] }, + "node_modules/inflation": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/inflation/-/inflation-2.0.0.tgz", + "integrity": "sha1-i0F+R8KPklpFEz2RTKH9OJEH8w8=", + "engines": { + "node": ">= 0.8.0" + } + }, "node_modules/inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -835,6 +943,18 @@ "node": "^4.8.4 || ^6.10.1 || ^7.10.1 || >= 8.1.4" } }, + "node_modules/koa-bodyparser": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/koa-bodyparser/-/koa-bodyparser-4.3.0.tgz", + "integrity": "sha512-uyV8G29KAGwZc4q/0WUAjH+Tsmuv9ImfBUF2oZVyZtaeo0husInagyn/JH85xMSxM0hEk/mbCII5ubLDuqW/Rw==", + "dependencies": { + "co-body": "^6.0.0", + "copy-to": "^2.0.1" + }, + "engines": { + "node": ">=8.0.0" + } + }, "node_modules/koa-compose": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz", @@ -871,6 +991,21 @@ "node": ">=8" } }, + "node_modules/log4js": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.4.2.tgz", + "integrity": "sha512-k80cggS2sZQLBwllpT1p06GtfvzMmSdUCkW96f0Hj83rKGJDAu2vZjt9B9ag2vx8Zz1IXzxoLgqvRJCdMKybGg==", + "dependencies": { + "date-format": "^4.0.4", + "debug": "^4.3.3", + "flatted": "^3.2.5", + "rfdc": "^1.3.0", + "streamroller": "^3.0.4" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -992,6 +1127,14 @@ } } }, + "node_modules/object-inspect": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", + "integrity": "sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==", + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -1212,6 +1355,57 @@ "node": ">=8" } }, + "node_modules/qs": { + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "dependencies": { + "side-channel": "^1.0.4" + }, + "engines": { + "node": ">=0.6" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, + "node_modules/raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "dependencies": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "dependencies": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + }, + "engines": { + "node": ">= 0.8" + } + }, + "node_modules/raw-body/node_modules/statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -1225,6 +1419,11 @@ "node": ">= 6" } }, + "node_modules/rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, "node_modules/rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -1258,6 +1457,11 @@ } ] }, + "node_modules/safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "node_modules/setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -1296,6 +1500,19 @@ "node": ">=0.10.0" } }, + "node_modules/side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "dependencies": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + }, + "funding": { + "url": "https://github.com/sponsors/ljharb" + } + }, "node_modules/statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", @@ -1304,6 +1521,19 @@ "node": ">= 0.6" } }, + "node_modules/streamroller": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-3.0.5.tgz", + "integrity": "sha512-5uzTEUIi4OB5zy/H30kbUN/zpDNJsFUA+Z47ZL8EfrP93lcZvRLEqdbhdunEPa7CouuAzXXsHpCJ9dg90Umw7g==", + "dependencies": { + "date-format": "^4.0.5", + "debug": "^4.3.3", + "fs-extra": "^10.0.1" + }, + "engines": { + "node": ">=8.0" + } + }, "node_modules/string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -1356,6 +1586,11 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" }, + "node_modules/tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" + }, "node_modules/tsscmp": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", @@ -1393,6 +1628,14 @@ "node": ">= 10.0.0" } }, + "node_modules/unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=", + "engines": { + "node": ">= 0.8" + } + }, "node_modules/util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", @@ -1682,6 +1925,14 @@ "resolved": "https://registry.npmjs.org/arr-union/-/arr-union-3.1.0.tgz", "integrity": "sha1-45sJrqne+Gao8gbiiK9jkZuuOcQ=" }, + "async-mutex": { + "version": "0.3.2", + "resolved": "https://registry.npmjs.org/async-mutex/-/async-mutex-0.3.2.tgz", + "integrity": "sha512-HuTK7E7MT7jZEh1P9GtRW9+aTWiDWWi9InbZ5hjxrnRa39KS4BW04+xLBhYNS2aXhHUIKZSw3gj4Pn1pj+qGAA==", + "requires": { + "tslib": "^2.3.1" + } + }, "balanced-match": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/balanced-match/-/balanced-match-1.0.2.tgz", @@ -1725,6 +1976,11 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, + "bytes": { + "version": "3.1.2", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.2.tgz", + "integrity": "sha512-/Nf7TyzTx6S3yRJObOAV7956r8cr2+Oj8AC5dt8wSP3BQAoeX58NoHyCU8P8zGkNXStjTSi6fzO6F0pBdcYbEg==" + }, "cache-content-type": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz", @@ -1734,6 +1990,15 @@ "ylru": "^1.2.0" } }, + "call-bind": { + "version": "1.0.2", + "resolved": "https://registry.npmjs.org/call-bind/-/call-bind-1.0.2.tgz", + "integrity": "sha512-7O+FbCihrB5WGbFYesctwmTKae6rOiIzmz1icreWJ+0aA7LJfuqhEso2T9ncpcFtzMQtzXf2QGGueWJGTYsqrA==", + "requires": { + "function-bind": "^1.1.1", + "get-intrinsic": "^1.0.2" + } + }, "chownr": { "version": "1.1.4", "resolved": "https://registry.npmjs.org/chownr/-/chownr-1.1.4.tgz", @@ -1756,6 +2021,17 @@ "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, + "co-body": { + "version": "6.1.0", + "resolved": "https://registry.npmjs.org/co-body/-/co-body-6.1.0.tgz", + "integrity": "sha512-m7pOT6CdLN7FuXUcpuz/8lfQ/L77x8SchHCF4G0RBTJO20Wzmhn5Sp4/5WsKy8OSpifBSUrmg83qEqaDHdyFuQ==", + "requires": { + "inflation": "^2.0.0", + "qs": "^6.5.2", + "raw-body": "^2.3.3", + "type-is": "^1.6.16" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -1783,6 +2059,11 @@ "keygrip": "~1.1.0" } }, + "copy-to": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/copy-to/-/copy-to-2.0.1.tgz", + "integrity": "sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU=" + }, "cross-fetch": { "version": "3.1.5", "resolved": "https://registry.npmjs.org/cross-fetch/-/cross-fetch-3.1.5.tgz", @@ -1791,6 +2072,11 @@ "node-fetch": "2.6.7" } }, + "date-format": { + "version": "4.0.5", + "resolved": "https://registry.npmjs.org/date-format/-/date-format-4.0.5.tgz", + "integrity": "sha512-zBhRiN/M0gDxUoM2xRtzTjJzSg0XEi1ofYpF84PfXeS3hN2PsGxmc7jw3DNQtFlimRbMmob5FC3G0cJq6jQQpw==" + }, "debug": { "version": "4.3.3", "resolved": "https://registry.npmjs.org/debug/-/debug-4.3.3.tgz", @@ -1880,6 +2166,11 @@ "path-exists": "^4.0.0" } }, + "flatted": { + "version": "3.2.5", + "resolved": "https://registry.npmjs.org/flatted/-/flatted-3.2.5.tgz", + "integrity": "sha512-WIWGi2L3DyTUvUrwRKgGi9TwxQMUEqPOPQBVi71R96jZXJdFskXEmf54BoZaS1kknGODoIGASGEzBUYdyMCBJg==" + }, "for-in": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/for-in/-/for-in-1.0.2.tgz", @@ -1918,6 +2209,21 @@ "resolved": "https://registry.npmjs.org/fs.realpath/-/fs.realpath-1.0.0.tgz", "integrity": "sha1-FQStJSMVjKpA20onh8sBQRmU6k8=" }, + "function-bind": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/function-bind/-/function-bind-1.1.1.tgz", + "integrity": "sha512-yIovAzMX49sF8Yl58fSCWJ5svSLuaibPxXQJFLmBObTuCr0Mf1KiPopGM9NiFjiYBCbfaa2Fh6breQ6ANVTI0A==" + }, + "get-intrinsic": { + "version": "1.1.1", + "resolved": "https://registry.npmjs.org/get-intrinsic/-/get-intrinsic-1.1.1.tgz", + "integrity": "sha512-kWZrnVM42QCiEA2Ig1bG8zjoIMOgxWwYCEeNdwY6Tv/cOSeGpcoX4pXHfKUxNKVoArnrEr2e9srnAxxGIraS9Q==", + "requires": { + "function-bind": "^1.1.1", + "has": "^1.0.3", + "has-symbols": "^1.0.1" + } + }, "get-stream": { "version": "5.2.0", "resolved": "https://registry.npmjs.org/get-stream/-/get-stream-5.2.0.tgz", @@ -1944,6 +2250,14 @@ "resolved": "https://registry.npmjs.org/graceful-fs/-/graceful-fs-4.2.9.tgz", "integrity": "sha512-NtNxqUcXgpW2iMrfqSfR73Glt39K+BLwWsPs94yR63v45T0Wbej7eRmL5cWfwEgqXnmjQp3zaJTshdRW/qC2ZQ==" }, + "has": { + "version": "1.0.3", + "resolved": "https://registry.npmjs.org/has/-/has-1.0.3.tgz", + "integrity": "sha512-f2dvO0VU6Oej7RkWJGrehjbzMAjFp5/VKPp5tTpWIV4JHHZK1/BxbFRtf/siA2SWTe09caDmVtYYzWEIbBS4zw==", + "requires": { + "function-bind": "^1.1.1" + } + }, "has-symbols": { "version": "1.0.3", "resolved": "https://registry.npmjs.org/has-symbols/-/has-symbols-1.0.3.tgz", @@ -1994,11 +2308,24 @@ "debug": "4" } }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "ieee754": { "version": "1.2.1", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.2.1.tgz", "integrity": "sha512-dcyqhDvX1C46lXZcVqCpK+FtMRQVdIMN6/Df5js2zouUsqG7I6sFxitIC+7KYK29KdXOLHdu9zL4sFnoVQnqaA==" }, + "inflation": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/inflation/-/inflation-2.0.0.tgz", + "integrity": "sha1-i0F+R8KPklpFEz2RTKH9OJEH8w8=" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -2099,6 +2426,15 @@ "vary": "^1.1.2" } }, + "koa-bodyparser": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/koa-bodyparser/-/koa-bodyparser-4.3.0.tgz", + "integrity": "sha512-uyV8G29KAGwZc4q/0WUAjH+Tsmuv9ImfBUF2oZVyZtaeo0husInagyn/JH85xMSxM0hEk/mbCII5ubLDuqW/Rw==", + "requires": { + "co-body": "^6.0.0", + "copy-to": "^2.0.1" + } + }, "koa-compose": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz", @@ -2126,6 +2462,18 @@ "p-locate": "^4.1.0" } }, + "log4js": { + "version": "6.4.2", + "resolved": "https://registry.npmjs.org/log4js/-/log4js-6.4.2.tgz", + "integrity": "sha512-k80cggS2sZQLBwllpT1p06GtfvzMmSdUCkW96f0Hj83rKGJDAu2vZjt9B9ag2vx8Zz1IXzxoLgqvRJCdMKybGg==", + "requires": { + "date-format": "^4.0.4", + "debug": "^4.3.3", + "flatted": "^3.2.5", + "rfdc": "^1.3.0", + "streamroller": "^3.0.4" + } + }, "media-typer": { "version": "0.3.0", "resolved": "https://registry.npmjs.org/media-typer/-/media-typer-0.3.0.tgz", @@ -2211,6 +2559,11 @@ "whatwg-url": "^5.0.0" } }, + "object-inspect": { + "version": "1.12.0", + "resolved": "https://registry.npmjs.org/object-inspect/-/object-inspect-1.12.0.tgz", + "integrity": "sha512-Ho2z80bVIvJloH+YzRmpZVQe87+qASmBUKZDWgx9cu+KDrX2ZDH/3tMy+gXbZETVGs2M8YdxObOh7XAtim9Y0g==" + }, "on-finished": { "version": "2.4.1", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.4.1.tgz", @@ -2376,6 +2729,44 @@ "puppeteer-extra-plugin-user-data-dir": "^2.3.1" } }, + "qs": { + "version": "6.10.3", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.10.3.tgz", + "integrity": "sha512-wr7M2E0OFRfIfJZjKGieI8lBKb7fRCH4Fv5KNPEs7gJ8jadvotdsS08PzOKR7opXhZ/Xkjtt3WF9g38drmyRqQ==", + "requires": { + "side-channel": "^1.0.4" + } + }, + "raw-body": { + "version": "2.5.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.5.1.tgz", + "integrity": "sha512-qqJBtEyVgS0ZmPGdCFPWJ3FreoqvG4MVQln/kCgF7Olq95IbOp0/BWyMwbdtn4VTvkM8Y7khCQ2Xgk/tcrCXig==", + "requires": { + "bytes": "3.1.2", + "http-errors": "2.0.0", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + }, + "dependencies": { + "http-errors": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/http-errors/-/http-errors-2.0.0.tgz", + "integrity": "sha512-FtwrG/euBzaEjYeRqOgly7G0qviiXoJWnvEH2Z1plBdXgbyjv34pHTSb9zoeHMyDy33+DWy5Wt9Wo+TURtOYSQ==", + "requires": { + "depd": "2.0.0", + "inherits": "2.0.4", + "setprototypeof": "1.2.0", + "statuses": "2.0.1", + "toidentifier": "1.0.1" + } + }, + "statuses": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/statuses/-/statuses-2.0.1.tgz", + "integrity": "sha512-RwNA9Z/7PrK06rYLIzFMlaF+l73iwpzsqRIFgbMLbTcLD6cOao82TaWefPXQvB2fOC4AjuYSEndS7N/mTCbkdQ==" + } + } + }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -2386,6 +2777,11 @@ "util-deprecate": "^1.0.1" } }, + "rfdc": { + "version": "1.3.0", + "resolved": "https://registry.npmjs.org/rfdc/-/rfdc-1.3.0.tgz", + "integrity": "sha512-V2hovdzFbOi77/WajaSMXk2OLm+xNIeQdMMuB7icj7bk6zi2F8GGAxigcnDFpJHbNyNcgyJDiP+8nOrY5cZGrA==" + }, "rimraf": { "version": "3.0.2", "resolved": "https://registry.npmjs.org/rimraf/-/rimraf-3.0.2.tgz", @@ -2399,6 +2795,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.2.1.tgz", "integrity": "sha512-rp3So07KcdmmKbGvgaNxQSJr7bGVSVk5S9Eq1F+ppbRo70+YeaDxkw5Dd8NPN+GD6bjnYm2VuPuCXmpuYvmCXQ==" }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "setprototypeof": { "version": "1.2.0", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.2.0.tgz", @@ -2430,11 +2831,31 @@ } } }, + "side-channel": { + "version": "1.0.4", + "resolved": "https://registry.npmjs.org/side-channel/-/side-channel-1.0.4.tgz", + "integrity": "sha512-q5XPytqFEIKHkGdiMIrY10mvLRvnQh42/+GoBlFW3b2LXLE2xxJpZFdm94we0BaoV3RwJyGqg5wS7epxTv0Zvw==", + "requires": { + "call-bind": "^1.0.0", + "get-intrinsic": "^1.0.2", + "object-inspect": "^1.9.0" + } + }, "statuses": { "version": "1.5.0", "resolved": "https://registry.npmjs.org/statuses/-/statuses-1.5.0.tgz", "integrity": "sha1-Fhx9rBd2Wf2YEfQ3cfqZOBR4Yow=" }, + "streamroller": { + "version": "3.0.5", + "resolved": "https://registry.npmjs.org/streamroller/-/streamroller-3.0.5.tgz", + "integrity": "sha512-5uzTEUIi4OB5zy/H30kbUN/zpDNJsFUA+Z47ZL8EfrP93lcZvRLEqdbhdunEPa7CouuAzXXsHpCJ9dg90Umw7g==", + "requires": { + "date-format": "^4.0.5", + "debug": "^4.3.3", + "fs-extra": "^10.0.1" + } + }, "string_decoder": { "version": "1.3.0", "resolved": "https://registry.npmjs.org/string_decoder/-/string_decoder-1.3.0.tgz", @@ -2481,6 +2902,11 @@ "resolved": "https://registry.npmjs.org/tr46/-/tr46-0.0.3.tgz", "integrity": "sha1-gYT9NH2snNwYWZLzpmIuFLnZq2o=" }, + "tslib": { + "version": "2.3.1", + "resolved": "https://registry.npmjs.org/tslib/-/tslib-2.3.1.tgz", + "integrity": "sha512-77EbyPPpMz+FRFRuAFlWMtmgUWGe9UOG2Z25NqCwiIjRhOf5iKGuzSe5P2w1laq+FkRy4p+PCuVkJSGkzTEKVw==" + }, "tsscmp": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/tsscmp/-/tsscmp-1.0.6.tgz", @@ -2509,6 +2935,11 @@ "resolved": "https://registry.npmjs.org/universalify/-/universalify-2.0.0.tgz", "integrity": "sha512-hAZsKq7Yy11Zu1DE0OzWjw7nnLZmJZYTDZZyEFHZdUhV8FkH5MCfoU1XMaxXovpyW5nq5scPqq0ZDP9Zyl04oQ==" }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index b3ca94e..2f8ffa4 100644 --- a/package.json +++ b/package.json @@ -2,17 +2,20 @@ "name": "rt-pupflare", "version": "1.0.0", "description": "", - "main": "index.js", + "main": "app.js", "scripts": { "test": "echo \"Error: no test specified\" && exit 1", - "start": "node index.js" + "start": "node app.js" }, "keywords": [], "author": "", "license": "ISC", "dependencies": { "@koa/router": "^10.1.1", + "async-mutex": "^0.3.2", "koa": "^2.13.4", + "koa-bodyparser": "^4.3.0", + "log4js": "^6.4.2", "minimist": "^1.2.5", "puppeteer": "^13.4.1", "puppeteer-extra": "^3.2.3", diff --git a/systemd/rt-pupflare@.service b/systemd/rt-pupflare@.service index b96a902..c2362f5 100644 --- a/systemd/rt-pupflare@.service +++ b/systemd/rt-pupflare@.service @@ -10,7 +10,7 @@ Restart=on-failure ProtectSystem=full PrivateDevices=true WorkingDirectory=/opt/thirdparty -ExecStart=/usr/bin/node $INSTALL_DIR/index.js $PARAMS +ExecStart=/usr/bin/node $INSTALL_DIR/app.js $PARAMS [Install] WantedBy=multi-user.target |