add ability to post data + handle downloads

3 files changed, 140 insertions, 6 deletions

diff --git a/index.js b/index.js
index 06dfac5..af7425b 100644
--- a/index.js
+++ b/index.js
@@ -4,12 +4,16 @@ const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker');
 puppeteer.use(AdblockerPlugin());
 puppeteer.use(StealthPlugin());
 const Koa = require('koa');
+const bodyParser = require('koa-bodyparser');
 const app = new Koa();
+app.use(bodyParser());
+const jsesc = require('jsesc');
 
 const headersToRemove = [
     "host", "user-agent", "accept", "accept-encoding", "content-length",
     "forwarded", "x-forwarded-proto", "x-forwarded-for", "x-cloud-trace-context"
 ];
+const responseHeadersToRemove = ["Accept-Ranges", "Content-Length", "Keep-Alive", "Connection", "content-encoding"];
 
 (async () => {
     let options = {
@@ -22,17 +26,71 @@ const headersToRemove = [
     app.use(async ctx => {
         if (ctx.query.url) {
             const url = ctx.url.replace("/?url=", "");
+            let responseBody;
+            let responseHeaders;
             const page = await browser.newPage();
+            if (ctx.method == "POST") {
+                await page.removeAllListeners('request');
+                await page.setRequestInterception(true);
+                page.on('request', interceptedRequest => {
+                    var data = {
+                        'method': 'POST',
+                        'postData': ctx.request.rawBody
+                    };
+                    interceptedRequest.continue(data);
+                });
+            }
+            const client = await page.target().createCDPSession();
+            await client.send('Network.setRequestInterception', {
+                patterns: [{
+                    urlPattern: '*',
+                    resourceType: 'Document',
+                    interceptionStage: 'HeadersReceived'
+                }],
+            });
+
+            await client.on('Network.requestIntercepted', async e => {
+                let obj = { interceptionId: e.interceptionId };
+                if (e.isDownload) {
+                    await client.send('Network.getResponseBodyForInterception', {
+                        interceptionId: e.interceptionId
+                    }).then((result) => {
+                        if (result.base64Encoded) {
+                            responseBody = Buffer.from(result.body, 'base64');
+                        }
+                    });
+                    obj['errorReason'] = 'BlockedByClient';
+                    responseHeaders = e.responseHeaders;
+                }
+                await client.send('Network.continueInterceptedRequest', obj);
+                if (e.isDownload)
+                    await page.close();
+            });
             let headers = ctx.headers;
             headersToRemove.forEach(header => {
                 delete headers[header];
             });
-            await page.setExtraHTTPHeaders(headers);            
-            await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' });
-            if ((await page.content()).includes("cf-browser-verification"))
-                await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' });
-            ctx.body = await page.content();
-            await page.close();
+            await page.setExtraHTTPHeaders(headers);
+            try {
+                let response;
+                response = await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' });
+                if ((await page.content()).includes("cf-browser-verification"))
+                    response = await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' });
+                responseBody = await page.content();
+                responseHeaders = response.headers();
+                await page.close();
+            } catch (error) {
+                if (!error.toString().includes("ERR_BLOCKED_BY_CLIENT"))
+                    throw (error);
+            }
+            responseHeadersToRemove.forEach(header => {
+               delete responseHeaders[header];
+            });
+            Object.keys(responseHeaders).forEach(header => {
+               ctx.set(header, jsesc(responseHeaders[header]));
+            });
+            ctx.response.headers = responseHeaders;
+            ctx.body = responseBody;
         }
         else {
             ctx.body = "Please specify the URL in the 'url' query string.";
diff --git a/package-lock.json b/package-lock.json
index 5f7512a..966b28e 100644
--- a/package-lock.json
+++ b/package-lock.json
@@ -216,6 +216,11 @@
       "resolved": "https://registry.npmjs.org/buffer-crc32/-/buffer-crc32-0.2.13.tgz",
       "integrity": "sha1-DTM+PwDqxQqhRUq9MO+MKl2ackI="
     },
+    "bytes": {
+      "version": "3.1.0",
+      "resolved": "https://registry.npmjs.org/bytes/-/bytes-3.1.0.tgz",
+      "integrity": "sha512-zauLjrfCG+xvoyaqLoV8bLVXXNGC4JqlxFCutSDWA6fJrTo2ZuvLYTqZ7aHBLZSMOopbzwv8f+wZcVzfVTI2Dg=="
+    },
     "cache-content-type": {
       "version": "1.0.1",
       "resolved": "https://registry.npmjs.org/cache-content-type/-/cache-content-type-1.0.1.tgz",
@@ -247,6 +252,17 @@
       "resolved": "https://registry.npmjs.org/co/-/co-4.6.0.tgz",
       "integrity": "sha1-bqa989hTrlTMuOR7+gvz+QMfsYQ="
     },
+    "co-body": {
+      "version": "6.0.0",
+      "resolved": "https://registry.npmjs.org/co-body/-/co-body-6.0.0.tgz",
+      "integrity": "sha512-9ZIcixguuuKIptnY8yemEOuhb71L/lLf+Rl5JfJEUiDNJk0e02MBt7BPxR2GEh5mw8dPthQYR4jPI/BnS1MQgw==",
+      "requires": {
+        "inflation": "^2.0.0",
+        "qs": "^6.5.2",
+        "raw-body": "^2.3.3",
+        "type-is": "^1.6.16"
+      }
+    },
     "concat-map": {
       "version": "0.0.1",
       "resolved": "https://registry.npmjs.org/concat-map/-/concat-map-0.0.1.tgz",
@@ -281,6 +297,11 @@
         }
       }
     },
+    "copy-to": {
+      "version": "2.0.1",
+      "resolved": "https://registry.npmjs.org/copy-to/-/copy-to-2.0.1.tgz",
+      "integrity": "sha1-JoD7uAaKSNCGVrYJgJK9r8kG9KU="
+    },
     "debug": {
       "version": "3.1.0",
       "resolved": "https://registry.npmjs.org/debug/-/debug-3.1.0.tgz",
@@ -465,11 +486,24 @@
         }
       }
     },
+    "iconv-lite": {
+      "version": "0.4.24",
+      "resolved": "https://registry.npmjs.org/iconv-lite/-/iconv-lite-0.4.24.tgz",
+      "integrity": "sha512-v3MXnZAcvnywkTUEZomIActle7RXXeedOR31wwl7VlyoXO4Qi9arvSenNQWne1TcRwhCL1HwLI21bEqdpj8/rA==",
+      "requires": {
+        "safer-buffer": ">= 2.1.2 < 3"
+      }
+    },
     "ieee754": {
       "version": "1.1.13",
       "resolved": "https://registry.npmjs.org/ieee754/-/ieee754-1.1.13.tgz",
       "integrity": "sha512-4vf7I2LYV/HaWerSo3XmlMkp5eZ83i+/CDluXi/IGTs/O1sejBNhTtnxzmRZfvOUqj7lZjqHkeTvpgSFDlWZTg=="
     },
+    "inflation": {
+      "version": "2.0.0",
+      "resolved": "https://registry.npmjs.org/inflation/-/inflation-2.0.0.tgz",
+      "integrity": "sha1-i0F+R8KPklpFEz2RTKH9OJEH8w8="
+    },
     "inflight": {
       "version": "1.0.6",
       "resolved": "https://registry.npmjs.org/inflight/-/inflight-1.0.6.tgz",
@@ -512,6 +546,11 @@
       "resolved": "https://registry.npmjs.org/isobject/-/isobject-3.0.1.tgz",
       "integrity": "sha1-TkMekrEalzFjaqH5yNHMvP2reN8="
     },
+    "jsesc": {
+      "version": "3.0.1",
+      "resolved": "https://registry.npmjs.org/jsesc/-/jsesc-3.0.1.tgz",
+      "integrity": "sha512-w+MMxnByppM4jwskitZotEtvtO3a2C7WOz31NxJToGisHuysCAQQU7umb/pA/6soPFe8LGjXFEFbuPuLEPm7Ag=="
+    },
     "keygrip": {
       "version": "1.1.0",
       "resolved": "https://registry.npmjs.org/keygrip/-/keygrip-1.1.0.tgz",
@@ -558,6 +597,15 @@
         "vary": "^1.1.2"
       }
     },
+    "koa-bodyparser": {
+      "version": "4.3.0",
+      "resolved": "https://registry.npmjs.org/koa-bodyparser/-/koa-bodyparser-4.3.0.tgz",
+      "integrity": "sha512-uyV8G29KAGwZc4q/0WUAjH+Tsmuv9ImfBUF2oZVyZtaeo0husInagyn/JH85xMSxM0hEk/mbCII5ubLDuqW/Rw==",
+      "requires": {
+        "co-body": "^6.0.0",
+        "copy-to": "^2.0.1"
+      }
+    },
     "koa-compose": {
       "version": "4.1.0",
       "resolved": "https://registry.npmjs.org/koa-compose/-/koa-compose-4.1.0.tgz",
@@ -853,6 +901,22 @@
         }
       }
     },
+    "qs": {
+      "version": "6.9.4",
+      "resolved": "https://registry.npmjs.org/qs/-/qs-6.9.4.tgz",
+      "integrity": "sha512-A1kFqHekCTM7cz0udomYUoYNWjBebHm/5wzU/XqrBRBNWectVH0QIiN+NEcZ0Dte5hvzHwbr8+XQmguPhJ6WdQ=="
+    },
+    "raw-body": {
+      "version": "2.4.1",
+      "resolved": "https://registry.npmjs.org/raw-body/-/raw-body-2.4.1.tgz",
+      "integrity": "sha512-9WmIKF6mkvA0SLmA2Knm9+qj89e+j1zqgyn8aXGd7+nAduPoqgI9lO57SAZNn/Byzo5P7JhXTyg9PzaJbH73bA==",
+      "requires": {
+        "bytes": "3.1.0",
+        "http-errors": "1.7.3",
+        "iconv-lite": "0.4.24",
+        "unpipe": "1.0.0"
+      }
+    },
     "readable-stream": {
       "version": "3.6.0",
       "resolved": "https://registry.npmjs.org/readable-stream/-/readable-stream-3.6.0.tgz",
@@ -876,6 +940,11 @@
       "resolved": "https://registry.npmjs.org/safe-buffer/-/safe-buffer-5.1.2.tgz",
       "integrity": "sha512-Gd2UZBJDkXlY7GbJxfsE8/nvKkUEU1G38c1siN6QP6a9PT9MmHB8GnpscSmMJSoF8LOIrt8ud/wPtojys4G6+g=="
     },
+    "safer-buffer": {
+      "version": "2.1.2",
+      "resolved": "https://registry.npmjs.org/safer-buffer/-/safer-buffer-2.1.2.tgz",
+      "integrity": "sha512-YZo3K82SD7Riyi0E1EQPojLz7kpepnSQI9IyPbHHg1XXXevb5dJI7tpyN2ADxGcQbHG7vcyRHk0cbwqcQriUtg=="
+    },
     "setprototypeof": {
       "version": "1.1.1",
       "resolved": "https://registry.npmjs.org/setprototypeof/-/setprototypeof-1.1.1.tgz",
@@ -996,6 +1065,11 @@
         "through": "^2.3.8"
       }
     },
+    "unpipe": {
+      "version": "1.0.0",
+      "resolved": "https://registry.npmjs.org/unpipe/-/unpipe-1.0.0.tgz",
+      "integrity": "sha1-sr9O6FFKrmFltIF4KdIbLvSZBOw="
+    },
     "util-deprecate": {
       "version": "1.0.2",
       "resolved": "https://registry.npmjs.org/util-deprecate/-/util-deprecate-1.0.2.tgz",
diff --git a/package.json b/package.json
index 748f80f..bf7142c 100644
--- a/package.json
+++ b/package.json
@@ -11,7 +11,9 @@
   "author": "",
   "license": "ISC",
   "dependencies": {
+    "jsesc": "^3.0.1",
     "koa": "^2.12.0",
+    "koa-bodyparser": "^4.3.0",
     "puppeteer": "^3.1.0",
     "puppeteer-extra": "^3.1.9",
     "puppeteer-extra-plugin-adblocker": "^2.11.3",