From 2bfe081a7f582015f3e4f5cf9aa98c349182276f Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Sat, 23 May 2020 23:23:28 +0200 Subject: add ability to post data + handle downloads --- index.js | 70 +++++++++++++++++++++++++++++++++++++++++++++++----- package-lock.json | 74 +++++++++++++++++++++++++++++++++++++++++++++++++++++++ package.json | 2 ++ 3 files changed, 140 insertions(+), 6 deletions(-) diff --git a/index.js b/index.js index 06dfac5..af7425b 100644 --- a/index.js +++ b/index.js @@ -4,12 +4,16 @@ const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); puppeteer.use(AdblockerPlugin()); puppeteer.use(StealthPlugin()); const Koa = require('koa'); +const bodyParser = require('koa-bodyparser'); const app = new Koa(); +app.use(bodyParser()); +const jsesc = require('jsesc'); const headersToRemove = [ "host", "user-agent", "accept", "accept-encoding", "content-length", "forwarded", "x-forwarded-proto", "x-forwarded-for", "x-cloud-trace-context" ]; +const responseHeadersToRemove = ["Accept-Ranges", "Content-Length", "Keep-Alive", "Connection", "content-encoding"]; (async () => { let options = { @@ -22,17 +26,71 @@ const headersToRemove = [ app.use(async ctx => { if (ctx.query.url) { const url = ctx.url.replace("/?url=", ""); + let responseBody; + let responseHeaders; const page = await browser.newPage(); + if (ctx.method == "POST") { + await page.removeAllListeners('request'); + await page.setRequestInterception(true); + page.on('request', interceptedRequest => { + var data = { + 'method': 'POST', + 'postData': ctx.request.rawBody + }; + interceptedRequest.continue(data); + }); + } + const client = await page.target().createCDPSession(); + await client.send('Network.setRequestInterception', { + patterns: [{ + urlPattern: '*', + resourceType: 'Document', + interceptionStage: 'HeadersReceived' + }], + }); + + await client.on('Network.requestIntercepted', async e => { + let obj = { interceptionId: e.interceptionId }; + if (e.isDownload) { + await client.send('Network.getResponseBodyForInterception', { + interceptionId: e.interceptionId + }).then((result) => { + if (result.base64Encoded) { + responseBody = Buffer.from(result.body, 'base64'); + } + }); + obj['errorReason'] = 'BlockedByClient'; + responseHeaders = e.responseHeaders; + } + await client.send('Network.continueInterceptedRequest', obj); + if (e.isDownload) + await page.close(); + }); let headers = ctx.headers; headersToRemove.forEach(header => { delete headers[header]; }); - await page.setExtraHTTPHeaders(headers); - await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' }); - if ((await page.content()).includes("cf-browser-verification")) - await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' }); - ctx.body = await page.content(); - await page.close(); + await page.setExtraHTTPHeaders(headers); + try { + let response; + response = await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' }); + if ((await page.content()).includes("cf-browser-verification")) + response = await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' }); + responseBody = await page.content(); + responseHeaders = response.headers(); + await page.close(); + } catch (error) { + if (!error.toString().includes("ERR_BLOCKED_BY_CLIENT")) + throw (error); + } + responseHeadersToRemove.forEach(header => { + delete responseHeaders[header]; + }); + Object.keys(responseHeaders).forEach(header => { + ctx.set(header, jsesc(responseHeaders[header])); + }); + ctx.response.headers = responseHeaders; + ctx.body = responseBody; } else { ctx.body = "Please specify the URL in the 'url' query string."; diff --git a/package-lock.json b/package-lock.json index 5f7512a..966b28e 100644 --- a/package-lock.json +++ b/package-lock.json @@ -216,6 +216,11 @@ "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz", "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI=" }, + "bytes": { + "version": "3.1.0", + "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz", + "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg==" + }, "cache-content-type": { "version": "1.0.1", "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz", @@ -247,6 +252,17 @@ "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz", "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ=" }, + "co-body": { + "version": "6.0.0", + "resolved": "https://registry.npmjs.org/co-body/-/co-body-6.0.0.tgz", + "integrity": "sha512-9ZIcixguuuKIptnY8yemEOuhb71L/lLf+Rl5JfJEUiDNJk0e02MBt7BPxR2GEh5mw8dPthQYR4jPI/BnS1MQgw==", + "requires": { + "inflation": "^2.0.0", + "qs": "^6.5.2", + "raw-body": "^2.3.3", + "type-is": "^1.6.16" + } + }, "concat-map": { "version": "0.0.1", "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz", @@ -281,6 +297,11 @@ } } }, + "copy-to": { + "version": "2.0.1", + "resolved": "https://registry.npmjs.org/copy-to/-/copy-to-2.0.1.tgz", + "integrity": "sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU=" + }, "debug": { "version": "3.1.0", "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz", @@ -465,11 +486,24 @@ } } }, + "iconv-lite": { + "version": "0.4.24", + "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz", + "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==", + "requires": { + "safer-buffer": ">= 2.1.2 < 3" + } + }, "ieee754": { "version": "1.1.13", "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz", "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg==" }, + "inflation": { + "version": "2.0.0", + "resolved": "https://registry.npmjs.org/inflation/-/inflation-2.0.0.tgz", + "integrity": "sha1-i0F+R8KPklpFEz2RTKH9OJEH8w8=" + }, "inflight": { "version": "1.0.6", "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz", @@ -512,6 +546,11 @@ "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz", "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8=" }, + "jsesc": { + "version": "3.0.1", + "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.1.tgz", + "integrity": "sha512-w+MMxnByppM4jwskitZotEtvtO3a2C7WOz31NxJToGisHuysCAQQU7umb/pA/6soPFe8LGjXFEFbuPuLEPm7Ag==" + }, "keygrip": { "version": "1.1.0", "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz", @@ -558,6 +597,15 @@ "vary": "^1.1.2" } }, + "koa-bodyparser": { + "version": "4.3.0", + "resolved": "https://registry.npmjs.org/koa-bodyparser/-/koa-bodyparser-4.3.0.tgz", + "integrity": "sha512-uyV8G29KAGwZc4q/0WUAjH+Tsmuv9ImfBUF2oZVyZtaeo0husInagyn/JH85xMSxM0hEk/mbCII5ubLDuqW/Rw==", + "requires": { + "co-body": "^6.0.0", + "copy-to": "^2.0.1" + } + }, "koa-compose": { "version": "4.1.0", "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz", @@ -853,6 +901,22 @@ } } }, + "qs": { + "version": "6.9.4", + "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz", + "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ==" + }, + "raw-body": { + "version": "2.4.1", + "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.1.tgz", + "integrity": "sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA==", + "requires": { + "bytes": "3.1.0", + "http-errors": "1.7.3", + "iconv-lite": "0.4.24", + "unpipe": "1.0.0" + } + }, "readable-stream": { "version": "3.6.0", "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz", @@ -876,6 +940,11 @@ "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz", "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g==" }, + "safer-buffer": { + "version": "2.1.2", + "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz", + "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg==" + }, "setprototypeof": { "version": "1.1.1", "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz", @@ -996,6 +1065,11 @@ "through": "^2.3.8" } }, + "unpipe": { + "version": "1.0.0", + "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz", + "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw=" + }, "util-deprecate": { "version": "1.0.2", "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz", diff --git a/package.json b/package.json index 748f80f..bf7142c 100644 --- a/package.json +++ b/package.json @@ -11,7 +11,9 @@ "author": "", "license": "ISC", "dependencies": { + "jsesc": "^3.0.1", "koa": "^2.12.0", + "koa-bodyparser": "^4.3.0", "puppeteer": "^3.1.0", "puppeteer-extra": "^3.1.9", "puppeteer-extra-plugin-adblocker": "^2.11.3", -- cgit v1.2.3