From fe804f5b34c9b9289131b110877ca55cb5b38c93 Mon Sep 17 00:00:00 2001 From: Emilien Devos Date: Sun, 7 Jun 2020 15:33:37 +0200 Subject: retry if the verification page is presented again --- index.js | 10 +++++++--- 1 file changed, 7 insertions(+), 3 deletions(-) diff --git a/index.js b/index.js index c8661cd..3670639 100644 --- a/index.js +++ b/index.js @@ -15,7 +15,7 @@ const responseHeadersToRemove = ["Accept-Ranges", "Content-Length", "Keep-Alive" (async () => { let options = { - headless: true, + headless: false, args: ['--no-sandbox', '--disable-setuid-sandbox'] }; if (process.env.PUPPETEER_SKIP_CHROMIUM_DOWNLOAD) @@ -71,10 +71,14 @@ const responseHeadersToRemove = ["Accept-Ranges", "Content-Length", "Keep-Alive" await page.setExtraHTTPHeaders(headers); try { let response; + let tryCount = 0; response = await page.goto(url, { timeout: 30000, waitUntil: 'domcontentloaded' }); - if ((await page.content()).includes("cf-browser-verification")) - response = await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' }); responseBody = await response.text(); + while (responseBody.includes("cf-browser-verification") && tryCount <= 10) { + response = await page.waitForNavigation({ timeout: 30000, waitUntil: 'domcontentloaded' }); + responseBody = await response.text(); + tryCount++; + } responseHeaders = response.headers(); const cookies = await page.cookies(); if (cookies) -- cgit v1.2.3