1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
|
const cookiesStorage = require('./cookies-storage')
const browser = require('./browser')
const os = require('os')
const path = require('path')
const argv = require('minimist')(process.argv.slice(2), {
string: ['retries', 'timeout', 'cookies', 'port', 'proxy'],
boolean: ['no-sandbox', 'headful'],
stopEarly: true,
default: {
port: 3000,
retries: 10,
timeout: 30000,
cookies: path.join(os.homedir(), '.rt-pupflare-cookies.json')
}
})
const maxTryCount = parseInt(argv.retries)
const loadingTimeout = parseInt(argv.timeout)
const Koa = require('koa');
const Router = require('@koa/router');
const app = new Koa();
const router = new Router();
router.get('/request', async (ctx, next) => {
if (!ctx.query.url)
throw new Error('url not specified')
const myResult = {
binary: false,
headers: [],
data: ''
};
/*if (ctx.method === "POST") {
await page.removeAllListeners('request');
await page.setRequestInterception(true);
page.on('request', interceptedRequest => {
var data = {
'method': 'POST',
'postData': ctx.request.rawBody
};
interceptedRequest.continue(data);
});
}*/
let myError = false
await new Promise(async (resolve, reject) => {
const page = await browser.getPage(
(e) => e.isDownload === true,
(response, headers) => {
Object.assign(myResult, {
data: response.base64Encoded ? response.body : btoa(response.body),
binary: true,
headers
})
resolve()
})
try {
let tryCount = 0
let response = await page.goto(ctx.query.url, {
timeout: loadingTimeout,
waitUntil: 'domcontentloaded'
})
let body = await response.text()
while ((body.includes("cf-browser-verification") || body.includes('cf-captcha-container')) && tryCount <= maxTryCount) {
let newResponse = await page.waitForNavigation({
timeout: loadingTimeout,
waitUntil: 'domcontentloaded'
});
if (newResponse)
response = newResponse;
body = await response.text();
tryCount++;
}
myResult.data = await response.text()
myResult.headers = await response.headers()
resolve()
} catch (error) {
if (!error.toString().includes("ERR_BLOCKED_BY_CLIENT")) {
ctx.status = 500
ctx.body = error
resolve()
} else {
myError = true
}
}
})
if (!myError)
ctx.body = JSON.stringify(myResult)
await next()
})
.get('/cookies', async (ctx, next) => {
ctx.body = JSON.stringify(await cookiesStorage.get())
await next()
});
(async () => {
cookiesStorage.setFileName(argv.cookies)
// console.log(argv)
if (argv.proxy)
browser.setProxy(argv.proxy)
if (argv['no-sandbox'])
browser.disableSandbox()
if (argv.headful)
browser.setHeadful()
await browser.launch()
app.use(router.routes())
.use(router.allowedMethods())
app.on('error', (error) => {
console.error(error)
})
app.listen(parseInt(argv.port), '127.0.0.1')
})();
|
