From f6d2eb6a698a13c6bc2a800793c3331a52a5e47a Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Fri, 22 May 2020 22:25:17 +0200 Subject: add ability to pass headers --- index.js | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index 5a15868..06dfac5 100644 --- a/index.js +++ b/index.js @@ -6,6 +6,11 @@ puppeteer.use(StealthPlugin()); const Koa = require('koa'); const app = new Koa(); +const headersToRemove = [ + "host", "user-agent", "accept", "accept-encoding", "content-length", + "forwarded", "x-forwarded-proto", "x-forwarded-for", "x-cloud-trace-context" +]; + (async () => { let options = { headless: true, @@ -16,15 +21,21 @@ const app = new Koa(); const browser = await puppeteer.launch(options); app.use(async ctx => { if (ctx.query.url) { + const url = ctx.url.replace("/?url=", ""); const page = await browser.newPage(); - await page.goto(ctx.query.url, {timeout: 30000, waitUntil: 'domcontentloaded'}); + let headers = ctx.headers; + headersToRemove.forEach(header => { + delete headers[header]; + }); + await page.setExtraHTTPHeaders(headers); + await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' }); if ((await page.content()).includes("cf-browser-verification")) - await page.waitForNavigation({timeout: 30000, waitUntil: 'domcontentloaded'}); + await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' }); ctx.body = await page.content(); await page.close(); } else { - ctx.body = "Please specify the URL query string."; + ctx.body = "Please specify the URL in the 'url' query string."; } }); app.listen(3000); -- cgit v1.2.3