diff options
| -rw-r--r-- | index.js | 4 | ||||
| -rw-r--r-- | package-lock.json | 165 | ||||
| -rw-r--r-- | package.json | 1 |
3 files changed, 2 insertions, 168 deletions
@@ -1,7 +1,5 @@ const puppeteer = require('puppeteer-extra'); const StealthPlugin = require('puppeteer-extra-plugin-stealth'); -const AdblockerPlugin = require('puppeteer-extra-plugin-adblocker'); -puppeteer.use(AdblockerPlugin()); puppeteer.use(StealthPlugin()); const Koa = require('koa'); const bodyParser = require('koa-bodyparser'); @@ -76,7 +74,7 @@ const responseHeadersToRemove = ["Accept-Ranges", "Content-Length", "Keep-Alive" response = await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' }); if ((await page.content()).includes("cf-browser-verification")) response = await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' }); - responseBody = await page.content(); + responseBody = await response.text(); responseHeaders = response.headers(); const cookies = await page.cookies(); if (cookies) diff --git a/package-lock.json b/package-lock.json index 966b28e..01105ec 100644 --- a/package-lock.json +++ b/package-lock.json @@ -1,132 +1,14 @@ { - "name": "koa-puppeteer", + "name": "pupflare", "version": "1.0.0", "lockfileVersion": 1, "requires": true, "dependencies": { - "@cliqz/adblocker": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/@cliqz/adblocker/-/adblocker-1.14.4.tgz", - "integrity": "sha512-3lTnstcPzNytR/NJ1VS5wt3iztt0tgiPJqXeOhbp80G9XfXsWfj18n6IJryyillbkNyRlmi7ElS3s/DGAZ17Bw==", - "requires": { - "@remusao/guess-url-type": "^1.1.2", - "@remusao/small": "^1.1.2", - "@remusao/smaz": "^1.7.1", - "@types/chrome": "^0.0.107", - "@types/firefox-webext-browser": "^70.0.1", - "tldts-experimental": "^5.6.21" - }, - "dependencies": { - "@types/chrome": { - "version": "0.0.107", - "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.107.tgz", - "integrity": "sha512-7z/4YkJu4B8ROxZhbzIx5olbRJqzoDdAoMhSzcJlElsZzv2eTxsvxNkN/QFq0ClbAFNkFf07XVoTrWFHD12QyA==", - "requires": { - "@types/filesystem": "*", - "@types/har-format": "*" - } - } - } - }, - "@cliqz/adblocker-content": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/@cliqz/adblocker-content/-/adblocker-content-1.14.4.tgz", - "integrity": "sha512-xuQ93BoHKE2PbL8X1v2J7Eu9Ckgrh2rya9FnTgzQ1cyMn6vvtT4m/VSm/dt/JiRcKJbdMCyQA/1Z2GW0iVZz4A==" - }, - "@cliqz/adblocker-puppeteer": { - "version": "1.14.4", - "resolved": "https://registry.npmjs.org/@cliqz/adblocker-puppeteer/-/adblocker-puppeteer-1.14.4.tgz", - "integrity": "sha512-FhN2eVFtHDDoCGHpqF00NLfT+HtcfnkSPWx/baEJ0gNaWdNZTI8EQYrX/HFz2oj/KD6MHxDr7opnDUNRIO78Gg==", - "requires": { - "@cliqz/adblocker": "^1.14.4", - "@cliqz/adblocker-content": "^1.14.4", - "@types/puppeteer": "^2.0.0", - "tldts-experimental": "^5.6.21" - }, - "dependencies": { - "@types/puppeteer": { - "version": "2.1.1", - "resolved": "https://registry.npmjs.org/@types/puppeteer/-/puppeteer-2.1.1.tgz", - "integrity": "sha512-FqPZvUtnpTGrqbHvPUn76pvVcBPEVEqZftrdOjr6YRkaaxkjKQ8dQLNaQBjER7Lvd1Q6+0R0XR+N3tYGWBSzNw==", - "requires": { - "@types/node": "*" - } - } - } - }, - "@remusao/guess-url-type": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@remusao/guess-url-type/-/guess-url-type-1.1.2.tgz", - "integrity": "sha512-NDcy4J7bieEU2WYSn1LZKjaIG+5ORDWXA//6dXnppVYPmmJ9zjJ1vJVKUKdZ1mjvTp7WLzQH6QD/wygaZ13G1Q==" - }, - "@remusao/small": { - "version": "1.1.2", - "resolved": "https://registry.npmjs.org/@remusao/small/-/small-1.1.2.tgz", - "integrity": "sha512-cMteQVgRT+LhSTQQdR0pWlLRXOTV8FeePOxcwkrroYCKIWpqIKNvbimAEiioMlclDmSHos9Nu+Wz3iNIDsyQaQ==" - }, - "@remusao/smaz": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@remusao/smaz/-/smaz-1.8.0.tgz", - "integrity": "sha512-tpyluMIHQSCOX1VHBBrn7KP+S1m/fovJg9Tizo06qR9OYopYehDqkuBJGLnUthE3uhm7tbxb1XI2IN4tUjOFhw==", - "requires": { - "@remusao/smaz-compress": "^1.8.0", - "@remusao/smaz-decompress": "^1.8.0" - } - }, - "@remusao/smaz-compress": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@remusao/smaz-compress/-/smaz-compress-1.8.0.tgz", - "integrity": "sha512-gDlnWyYmDppwdVZZkBuN8/gzpcqxFwEEev8QxyArWbOn/1rIXMfnm+bwY/N80rmakHTWaybL0yNxTMJSoVXrUw==", - "requires": { - "@remusao/trie": "^1.3.0" - } - }, - "@remusao/smaz-decompress": { - "version": "1.8.0", - "resolved": "https://registry.npmjs.org/@remusao/smaz-decompress/-/smaz-decompress-1.8.0.tgz", - "integrity": "sha512-DDGL8wB2L1OUPbUEashqBQzuEVk1xmVOR+35lERkwl3Qg5UoiL//9GWSq38jq8XlTqUG5R+BY2dQa4XX2NDeeQ==" - }, - "@remusao/trie": { - "version": "1.3.0", - "resolved": "https://registry.npmjs.org/@remusao/trie/-/trie-1.3.0.tgz", - "integrity": "sha512-IykMLl0vr8L1sJzvtGR4izkNh1Ous2njaeC4NEvAcfALCdCo7Y4VpISU0I8DaZFU2lGAFKyeh2Jz4WaM/3hbEw==" - }, - "@types/chrome": { - "version": "0.0.91", - "resolved": "https://registry.npmjs.org/@types/chrome/-/chrome-0.0.91.tgz", - "integrity": "sha512-vNvo9lJkp1AvViWrUwe1bxhoMwr5dRZWlgr1DTuaNkz97LsG56lDX1sceWeZir2gRACJ5vdHtoRdVAvm8C75Ug==", - "requires": { - "@types/filesystem": "*" - } - }, "@types/debug": { "version": "4.1.5", "resolved": "https://registry.npmjs.org/@types/debug/-/debug-4.1.5.tgz", "integrity": "sha512-Q1y515GcOdTHgagaVFhHnIFQ38ygs/kmxdNpvpou+raI9UO3YZcHDngBSYKQklcKlvA7iuQlmIKbzvmxcOE9CQ==" }, - "@types/filesystem": { - "version": "0.0.29", - "resolved": "https://registry.npmjs.org/@types/filesystem/-/filesystem-0.0.29.tgz", - "integrity": "sha512-85/1KfRedmfPGsbK8YzeaQUyV1FQAvMPMTuWFQ5EkLd2w7szhNO96bk3Rh/SKmOfd9co2rCLf0Voy4o7ECBOvw==", - "requires": { - "@types/filewriter": "*" - } - }, - "@types/filewriter": { - "version": "0.0.28", - "resolved": "https://registry.npmjs.org/@types/filewriter/-/filewriter-0.0.28.tgz", - "integrity": "sha1-wFTor02d11205jq8dviFFocU1LM=" - }, - "@types/firefox-webext-browser": { - "version": "70.0.1", - "resolved": "https://registry.npmjs.org/@types/firefox-webext-browser/-/firefox-webext-browser-70.0.1.tgz", - "integrity": "sha512-hjHsTR9vKs+yikWbNS/s7TVCx15M/MEn+VYx47wtT/W/wORsIZDD75gfUfP7lkzi+IxRvKMQBB/5/wMFlfgvgQ==" - }, - "@types/har-format": { - "version": "1.2.4", - "resolved": "https://registry.npmjs.org/@types/har-format/-/har-format-1.2.4.tgz", - "integrity": "sha512-iUxzm1meBm3stxUMzRqgOVHjj4Kgpgu5w9fm4X7kPRfSgVRzythsucEN7/jtOo8SQzm+HfcxWWzJS0mJDH/3DQ==" - }, "@types/node": { "version": "14.0.5", "resolved": "https://registry.npmjs.org/@types/node/-/node-14.0.5.tgz", @@ -707,11 +589,6 @@ "resolved": "https://registry.npmjs.org/negotiator/-/negotiator-0.6.2.tgz", "integrity": "sha512-hZXc7K2e+PgeI1eDBe/10Ard4ekbfrrqG8Ep+8Jmf4JID2bNg7NvCPOZN+kfF574pFQI7mum2AUqDidoKqcTOw==" }, - "node-fetch": { - "version": "2.6.0", - "resolved": "https://registry.npmjs.org/node-fetch/-/node-fetch-2.6.0.tgz", - "integrity": "sha512-8dG4H5ujfvFiqDmVu9fQ5bOHUC15JMjMY/Zumv26oOvvVJjM67KF8koCWIabKQ1GJIa9r2mMZscBq/TbdOcmNA==" - }, "on-finished": { "version": "2.3.0", "resolved": "https://registry.npmjs.org/on-finished/-/on-finished-2.3.0.tgz", @@ -850,33 +727,6 @@ } } }, - "puppeteer-extra-plugin-adblocker": { - "version": "2.11.3", - "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-adblocker/-/puppeteer-extra-plugin-adblocker-2.11.3.tgz", - "integrity": "sha512-9upM0w1iUJ8Q8YoFveGVpQzZBCvB1EQkwz4WHYOEhMjCmI5lkAcxxwVnUQTIbxRS4kDLTK7M17d5o+YKRpfkog==", - "requires": { - "@cliqz/adblocker-puppeteer": "^1.4.0", - "@types/chrome": "0.0.91", - "debug": "^4.1.1", - "node-fetch": "^2.6.0", - "puppeteer-extra-plugin": "^3.1.4" - }, - "dependencies": { - "debug": { - "version": "4.1.1", - "resolved": "https://registry.npmjs.org/debug/-/debug-4.1.1.tgz", - "integrity": "sha512-pYAIzeRo8J6KPEaJ0VWOh5Pzkbw/RetuzehGM7QRRX5he4fPHx2rdKMB256ehJCkX+XRQm16eZLqLNS8RSZXZw==", - "requires": { - "ms": "^2.1.1" - } - }, - "ms": { - "version": "2.1.2", - "resolved": "https://registry.npmjs.org/ms/-/ms-2.1.2.tgz", - "integrity": "sha512-sGkPx+VjMtmA6MX27oA4FBFELFCZZ4S4XqeGOXCv68tT+jb3vk/RyaKWP0PTKyWtmLSM0b+adUTEvbs1PEaH2w==" - } - } - }, "puppeteer-extra-plugin-stealth": { "version": "2.4.9", "resolved": "https://registry.npmjs.org/puppeteer-extra-plugin-stealth/-/puppeteer-extra-plugin-stealth-2.4.9.tgz", @@ -1024,19 +874,6 @@ "resolved": "https://registry.npmjs.org/through/-/through-2.3.8.tgz", "integrity": "sha1-DdTJ/6q8NXlgsbckEV1+Doai4fU=" }, - "tldts-core": { - "version": "5.6.25", - "resolved": "https://registry.npmjs.org/tldts-core/-/tldts-core-5.6.25.tgz", - "integrity": "sha512-RejSlOmLrMofDgr94b7dqhzCmjwjaT+jzu5AEcuz+HhFVu+HX7s2/6EL1TaoHBSP4lF7Bby7XnyWDaOTNBMJqQ==" - }, - "tldts-experimental": { - "version": "5.6.25", - "resolved": "https://registry.npmjs.org/tldts-experimental/-/tldts-experimental-5.6.25.tgz", - "integrity": "sha512-rBHoYaTyjMGrszpz6vm53+7UxWW1FEXJTKDvp38GxYPgYGcX6OT48N3vga6oxddlNz8q6u7MlnE3tthQ2mAXLQ==", - "requires": { - "tldts-core": "^5.6.25" - } - }, "toidentifier": { "version": "1.0.0", "resolved": "https://registry.npmjs.org/toidentifier/-/toidentifier-1.0.0.tgz", diff --git a/package.json b/package.json index 9418e5f..f115b62 100644 --- a/package.json +++ b/package.json @@ -16,7 +16,6 @@ "koa-bodyparser": "^4.3.0", "puppeteer": "^3.1.0", "puppeteer-extra": "^3.1.9", - "puppeteer-extra-plugin-adblocker": "^2.11.3", "puppeteer-extra-plugin-stealth": "^2.4.9" } } |