import { AsyncService } from 'civkit/async-service'; import { singleton } from 'tsyringe'; import { Blob } from 'buffer'; import { Curl, CurlCode, CurlFeature, HeaderInfo, Browser, getChromeConfig } from '@nomagick/node-libcurl-impersonate'; import { parseString as parseSetCookieString } from 'set-cookie-parser'; import { ScrappingOptions } from './puppeteer'; import { GlobalLogger } from './logger'; import { AssertionFailureError } from 'civkit/civ-rpc'; import { FancyFile } from 'civkit/fancy-file'; import { ServiceBadAttemptError, ServiceBadApproachError, TargetFileTooLargeError } from './errors'; import { TempFileManager } from '../services/temp-file'; import _ from 'lodash'; import { PassThrough, Readable } from 'stream'; import { AsyncLocalContext } from './async-context'; import { BlackHoleDetector } from './blackhole-detector'; import { humanReadableDataSize } from 'civkit/readability'; export interface CURLScrappingOptions extends ScrappingOptions { method?: string; body?: string | Buffer; maxSize?: number; } export const REDIRECTION_CODES = new Set([301, 302, 303, 307, 308]); @singleton() export class CurlControl extends AsyncService { logger = this.globalLogger.child({ service: this.constructor.name }); chromeVersion: string = `136`; safariVersion: string = `537.36`; platform: string = `Linux`; ua: string = `Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/${this.safariVersion} (KHTML, like Gecko) Chrome/${this.chromeVersion}.0.0.0 Safari/${this.safariVersion}`; lifeCycleTrack = new WeakMap(); impersonateCfg?: ReturnType; constructor( protected globalLogger: GlobalLogger, protected tempFileManager: TempFileManager, protected asyncLocalContext: AsyncLocalContext, protected blackHoleDetector: BlackHoleDetector, ) { super(...arguments); } override async init() { await this.dependencyReady(); if (process.platform === 'darwin') { this.platform = `macOS`; } else if (process.platform === 'win32') { this.platform = `Windows`; } this.emit('ready'); } impersonateChrome(ua: string) { this.chromeVersion = ua.match(/Chrome\/(\d+)/)![1]; this.safariVersion = ua.match(/AppleWebKit\/([\d\.]+)/)![1]; this.ua = ua; this.impersonateCfg = getChromeConfig({ version: this.chromeVersion, fingerprint: { ja3: '772,4865-4866-4867-49195-49199-49196-49200-52393-52392-49171-49172-156-157-47-53,23-0-45-17613-65281-18-43-65037-5-51-16-10-27-13-35-11,4588-29-23-24,0', ja4: 't13d1514h2_002f,0035,009c,009d,1301,1302,1303,c013,c014,c02b,c02c,c02f,c030,cca8,cca9_0005,000a,000b,000d,0012,0017,001b,0023,002b,002d,0033,44cd,fe0d,ff01_0401,0403,0501,0503,0601,0804,0805,0806', akami: '1:65536;2:0;4:6291456;6:262144|15663105|0|m,a,s,p' } }); } curlImpersonateHeader(curl: Curl, headers?: object) { let uaPlatform = this.platform; if (this.ua.includes('Windows')) { uaPlatform = 'Windows'; } else if (this.ua.includes('Android')) { uaPlatform = 'Android'; } else if (this.ua.includes('iPhone') || this.ua.includes('iPad') || this.ua.includes('iPod')) { uaPlatform = 'iOS'; } else if (this.ua.includes('CrOS')) { uaPlatform = 'Chrome OS'; } else if (this.ua.includes('Macintosh')) { uaPlatform = 'macOS'; } const mixinHeaders: Record = { 'sec-ch-ua': `"Chromium";v="${this.chromeVersion}", "Google Chrome";v="${this.chromeVersion}", "Not.A/Brand";v="99"`, 'sec-ch-ua-mobile': '?0', 'sec-ch-ua-platform': `"${uaPlatform}"`, 'Upgrade-Insecure-Requests': '1', 'User-Agent': this.ua, 'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7', 'Sec-Fetch-Site': 'none', 'Sec-Fetch-Mode': 'navigate', 'Sec-Fetch-User': '?1', 'Sec-Fetch-Dest': 'document', // 'Accept-Encoding': 'gzip, deflate, br, zstd', 'Accept-Language': 'en-US,en;q=0.9', 'Priority': 'u=0, i' }; const headersCopy: Record = { ...headers }; for (const k of Object.keys(mixinHeaders)) { const lowerK = k.toLowerCase(); if (headersCopy[lowerK]) { mixinHeaders[k] = headersCopy[lowerK]; delete headersCopy[lowerK]; } } Object.assign(mixinHeaders, headersCopy); curl.setOpt(Curl.option.HTTPHEADER, Object.entries(mixinHeaders).flatMap(([k, v]) => { if (Array.isArray(v) && v.length) { return v.map((v2) => `${k}: ${v2}`); } return [`${k}: ${v}`]; })); return curl; } urlToStream(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { return new Promise<{ statusCode: number, statusText?: string, data?: Readable, headers: HeaderInfo[], }>((resolve, reject) => { const actx = this.asyncLocalContext.ctx; let contentType = ''; const curl = Curl.impersonate(this.impersonateCfg || Browser.Chrome); curl.enable(CurlFeature.StreamResponse); curl.setOpt('URL', urlToCrawl.toString()); curl.setOpt(Curl.option.FOLLOWLOCATION, false); curl.setOpt(Curl.option.SSL_VERIFYPEER, false); curl.setOpt(Curl.option.TIMEOUT_MS, crawlOpts?.timeoutMs || 30_000); curl.setOpt(Curl.option.CONNECTTIMEOUT_MS, 3_000); curl.setOpt(Curl.option.LOW_SPEED_LIMIT, 32768); curl.setOpt(Curl.option.LOW_SPEED_TIME, 5_000); if (crawlOpts?.method) { curl.setOpt(Curl.option.CUSTOMREQUEST, crawlOpts.method.toUpperCase()); } if (crawlOpts?.body) { // @ts-ignore curl.setOpt(Curl.option.POSTFIELDS, crawlOpts.body.toString()); } const headersToSet = { ...crawlOpts?.extraHeaders }; if (crawlOpts?.cookies?.length) { const cookieKv: Record = {}; for (const cookie of crawlOpts.cookies) { cookieKv[cookie.name] = cookie.value; } for (const cookie of crawlOpts.cookies) { if (cookie.maxAge && cookie.maxAge < 0) { delete cookieKv[cookie.name]; continue; } if (cookie.expires && cookie.expires < new Date()) { delete cookieKv[cookie.name]; continue; } if (cookie.secure && urlToCrawl.protocol !== 'https:') { delete cookieKv[cookie.name]; continue; } if (cookie.domain && !urlToCrawl.hostname.endsWith(cookie.domain)) { delete cookieKv[cookie.name]; continue; } if (cookie.path && !urlToCrawl.pathname.startsWith(cookie.path)) { delete cookieKv[cookie.name]; continue; } } const cookieChunks = Object.entries(cookieKv).map(([k, v]) => `${k}=${encodeURIComponent(v)}`); headersToSet.cookie ??= cookieChunks.join('; '); } if (crawlOpts?.referer) { headersToSet.referer ??= crawlOpts.referer; } if (crawlOpts?.overrideUserAgent) { headersToSet['user-agent'] ??= crawlOpts.overrideUserAgent; } this.curlImpersonateHeader(curl, headersToSet); if (crawlOpts?.proxyUrl) { const proxyUrlCopy = new URL(crawlOpts.proxyUrl); curl.setOpt(Curl.option.PROXY, proxyUrlCopy.href); curl.setOpt(Curl.option.CONNECTTIMEOUT_MS, 6_000); curl.setOpt(Curl.option.PROXY_CREDENTIAL_NO_REUSE, 1); } let curlStream: Readable | undefined; let streamRecvCounter = 0; curl.on('error', (err, errCode) => { this.asyncLocalContext.bridge(actx, () => { curl.close(); const err2 = this.digestCurlCode(errCode, err.message) || new AssertionFailureError(`Failed to access ${urlToCrawl.origin}: ${err.message}`); err2.cause ??= err; if (curlStream) { // For some reason, manually emitting error event is required for curlStream. if (errCode === CurlCode.CURLE_HTTP2_STREAM && streamRecvCounter) { this.logger.warn(`Curl(${errCode}) ${urlToCrawl.origin}: ${err}, but some data was received, not emitting error to the stream.`, { err, urlToCrawl }); curlStream.push(null); return; } curlStream.emit('error', err2); curlStream.destroy(err2); } else { this.logger.warn(`Curl(${errCode}) ${urlToCrawl.origin}: ${err}`, { err, urlToCrawl }); } reject(err2); }); }); curl.setOpt(Curl.option.MAXFILESIZE, 4 * 1024 * 1024 * 1024); // 4GB let status = -1; let statusText: string | undefined; let contentEncoding = ''; curl.once('end', () => { if (curlStream) { curlStream.once('end', () => curl.close()); return; } curl.close(); }); curl.on('stream', (stream, statusCode, headers) => { status = statusCode; for (const headerSet of (headers as HeaderInfo[])) { for (const [k, v] of Object.entries(headerSet)) { if (k.trim().endsWith(':')) { Reflect.set(headerSet, k.slice(0, k.indexOf(':')), v || ''); Reflect.deleteProperty(headerSet, k); continue; } if (v === undefined) { Reflect.set(headerSet, k, ''); continue; } if (k.toLowerCase() === 'content-type' && typeof v === 'string') { contentType = v.toLowerCase(); } } } const lastResHeaders = headers[headers.length - 1]; statusText = (lastResHeaders as HeaderInfo).result?.reason; for (const [k, v] of Object.entries(lastResHeaders)) { const kl = k.toLowerCase(); if (kl === 'content-type') { contentType = (v || '').toLowerCase(); } if (kl === 'content-encoding') { contentEncoding = (v || '').toLowerCase(); } if (contentType && contentEncoding) { break; } } curlStream = stream; stream.on('data', (buf) => { if (buf.length > 0) { streamRecvCounter += buf.length; } }); const passThrough = new PassThrough(); stream.pipe(passThrough); stream.once('error', (err) => { passThrough.destroy(err); }); stream = passThrough; stream.once('end', () => { this.asyncLocalContext.bridge(actx, () => { this.logger.debug(`CURL: [${statusCode}${statusText ? ` ${statusText}` : ''}] ${urlToCrawl.origin} ${humanReadableDataSize(streamRecvCounter)} ${contentType} ${contentEncoding}`, { statusCode, statusText, traffic: streamRecvCounter, contentType, contentEncoding }); }); }); if ( REDIRECTION_CODES.has(statusCode) && headers.some((h) => h.hasOwnProperty('Location') || h.hasOwnProperty('location')) ) { stream.resume(); stream.once('end', () => { resolve({ statusCode: status, statusText, data: undefined, headers: headers as HeaderInfo[], }); }); return; } resolve({ statusCode: status, statusText, data: stream, headers: headers as HeaderInfo[], }); }); curl.perform(); }); } async urlToFile(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { let leftRedirection = 6; let cookieRedirects = 0; let opts = { ...crawlOpts }; let nextHopUrl = urlToCrawl; const fakeHeaderInfos: HeaderInfo[] = []; do { const s = await this.urlToStream(nextHopUrl, opts); const r = { ...s } as { statusCode: number, statusText?: string, data?: FancyFile, headers: HeaderInfo[], }; const headers = r.headers[r.headers.length - 1]; const claimedSize = parseInt(headers['Content-Length'] || headers['content-length']); if (crawlOpts?.maxSize && claimedSize > crawlOpts.maxSize) { throw new TargetFileTooLargeError(`Target file is too large: ${humanReadableDataSize(claimedSize)} > ${humanReadableDataSize(crawlOpts.maxSize)}`); } if (r.data) { const fpath = this.tempFileManager.alloc(); const fancyFile = FancyFile.auto(r.data, fpath); this.tempFileManager.bindPathTo(fancyFile, fpath); r.data = fancyFile; } if (REDIRECTION_CODES.has(r.statusCode)) { fakeHeaderInfos.push(...r.headers); const location: string | undefined = headers.Location || headers.location; const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie']; if (setCookieHeader) { const cookieAssignments = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader]; const parsed = cookieAssignments.filter(Boolean).map((x) => parseSetCookieString(x, { decodeValues: true })); if (parsed.length) { opts.cookies = [...(opts.cookies || []), ...parsed]; } if (!location) { cookieRedirects += 1; } } if (!location && !setCookieHeader) { let sizeAcc = 0; s.data!.on('data', (chunk) => { sizeAcc += chunk.length; if (crawlOpts?.maxSize && sizeAcc > crawlOpts.maxSize) { s.data!.destroy(new TargetFileTooLargeError(`Max size exceeded: ${humanReadableDataSize(sizeAcc)}`)); } }); // Follow curl behavior return { statusCode: r.statusCode, data: r.data, headers: fakeHeaderInfos.concat(r.headers), }; } if (!location && cookieRedirects > 1) { throw new ServiceBadApproachError(`Failed to access ${urlToCrawl}: Browser required to solve complex cookie preconditions.`); } nextHopUrl = new URL(location || '', nextHopUrl); leftRedirection -= 1; continue; } let sizeAcc = 0; s.data!.on('data', (chunk) => { sizeAcc += chunk.length; if (crawlOpts?.maxSize && sizeAcc > crawlOpts.maxSize) { s.data!.destroy(new TargetFileTooLargeError(`Max size exceeded: ${humanReadableDataSize(sizeAcc)}`)); } }); return { statusCode: r.statusCode, statusText: r.statusText, data: r.data, headers: fakeHeaderInfos.concat(r.headers), }; } while (leftRedirection > 0); throw new ServiceBadAttemptError(`Failed to access ${urlToCrawl}: Too many redirections.`); } async sideLoad(targetUrl: URL, crawlOpts?: CURLScrappingOptions) { const curlResult = await this.urlToFile(targetUrl, crawlOpts); this.blackHoleDetector.itWorked(); let finalURL = targetUrl; const sideLoadOpts: CURLScrappingOptions['sideLoad'] = { impersonate: {}, proxyOrigin: {}, }; for (const headers of curlResult.headers) { const potentialLastResult = sideLoadOpts.impersonate[finalURL.href]; sideLoadOpts.impersonate[finalURL.href] = { status: headers.result?.code || -1, headers: _.mergeWith(potentialLastResult?.headers || {}, _.omit(headers, 'result'), (objValue, srcValue) => { if (Array.isArray(objValue) && Array.isArray(srcValue)) { return [...objValue, ...srcValue]; } return undefined; }), contentType: headers['Content-Type'] || headers['content-type'], }; if (crawlOpts?.proxyUrl) { sideLoadOpts.proxyOrigin[finalURL.origin] = crawlOpts.proxyUrl; } if (headers.result?.code && [301, 302, 307, 308].includes(headers.result.code)) { const location = headers.Location || headers.location; if (location) { finalURL = new URL(location, finalURL); } } } const lastHeaders = curlResult.headers[curlResult.headers.length - 1]; const contentType = (lastHeaders['Content-Type'] || lastHeaders['content-type'])?.toLowerCase() || (await curlResult.data?.mimeType) || 'application/octet-stream'; const contentDisposition = lastHeaders['Content-Disposition'] || lastHeaders['content-disposition']; const fileName = contentDisposition?.match(/filename="([^"]+)"/i)?.[1] || finalURL.pathname.split('/').pop(); if (sideLoadOpts.impersonate[finalURL.href] && (await curlResult.data?.size)) { sideLoadOpts.impersonate[finalURL.href].body = curlResult.data; } // This should keep the file from being garbage collected and deleted until this asyncContext/request is done. this.lifeCycleTrack.set(this.asyncLocalContext.ctx, curlResult.data); return { finalURL, sideLoadOpts, chain: curlResult.headers, status: curlResult.statusCode, statusText: curlResult.statusText, headers: lastHeaders, contentType, contentDisposition, fileName, file: curlResult.data }; } async urlToBlob(urlToCrawl: URL, crawlOpts?: CURLScrappingOptions) { let leftRedirection = 6; let cookieRedirects = 0; let opts = { ...crawlOpts }; let nextHopUrl = urlToCrawl; const fakeHeaderInfos: HeaderInfo[] = []; do { const s = await this.urlToStream(nextHopUrl, opts); const r = { ...s } as { statusCode: number, statusText?: string, data?: Blob, headers: HeaderInfo[], }; const headers = r.headers[r.headers.length - 1]; const claimedSize = parseInt(headers['Content-Length'] || headers['content-length']); if (crawlOpts?.maxSize && claimedSize > crawlOpts.maxSize) { throw new TargetFileTooLargeError(`Target file is too large: ${humanReadableDataSize(claimedSize)} > ${humanReadableDataSize(crawlOpts.maxSize)}`); } if (REDIRECTION_CODES.has(r.statusCode)) { fakeHeaderInfos.push(...r.headers); const location: string | undefined = headers.Location || headers.location; const setCookieHeader = headers['Set-Cookie'] || headers['set-cookie']; if (setCookieHeader) { const cookieAssignments = Array.isArray(setCookieHeader) ? setCookieHeader : [setCookieHeader]; const parsed = cookieAssignments.filter(Boolean).map((x) => parseSetCookieString(x, { decodeValues: true })); if (parsed.length) { opts.cookies = [...(opts.cookies || []), ...parsed]; } if (!location) { cookieRedirects += 1; } } if (!location && !setCookieHeader) { // Follow curl behavior if (s.data) { const chunks: Buffer[] = []; let sizeAcc = 0; s.data.on('data', (chunk) => { chunks.push(chunk); sizeAcc += chunk.length; if (crawlOpts?.maxSize && sizeAcc > crawlOpts.maxSize) { s.data!.destroy(new TargetFileTooLargeError(`Max size exceeded: ${humanReadableDataSize(sizeAcc)}`)); } }); await new Promise((resolve, reject) => { s.data!.once('end', resolve); s.data!.once('error', reject); }); r.data = new Blob(chunks, { type: headers['Content-Type'] || headers['content-type'] }); } return { statusCode: r.statusCode, data: r.data, headers: fakeHeaderInfos.concat(r.headers), }; } if (!location && cookieRedirects > 1) { throw new ServiceBadApproachError(`Failed to access ${urlToCrawl}: Browser required to solve complex cookie preconditions.`); } nextHopUrl = new URL(location || '', nextHopUrl); leftRedirection -= 1; continue; } if (s.data) { const chunks: Buffer[] = []; let sizeAcc = 0; s.data.on('data', (chunk) => { chunks.push(chunk); sizeAcc += chunk.length; if (crawlOpts?.maxSize && sizeAcc > crawlOpts.maxSize) { s.data!.destroy(new TargetFileTooLargeError(`Max size exceeded: ${humanReadableDataSize(sizeAcc)}`)); } }); await new Promise((resolve, reject) => { s.data!.once('end', resolve); s.data!.once('error', reject); }); r.data = new Blob(chunks, { type: headers['Content-Type'] || headers['content-type'] }); } return { statusCode: r.statusCode, statusText: r.statusText, data: r.data, headers: fakeHeaderInfos.concat(r.headers), }; } while (leftRedirection > 0); throw new ServiceBadAttemptError(`Failed to access ${urlToCrawl}: Too many redirections.`); } async sideLoadBlob(targetUrl: URL, crawlOpts?: CURLScrappingOptions) { const curlResult = await this.urlToBlob(targetUrl, crawlOpts); this.blackHoleDetector.itWorked(); let finalURL = targetUrl; const sideLoadOpts: CURLScrappingOptions['sideLoad'] = { impersonate: {}, proxyOrigin: {}, }; for (const headers of curlResult.headers) { const potentialLastResult = sideLoadOpts.impersonate[finalURL.href]; sideLoadOpts.impersonate[finalURL.href] = { status: headers.result?.code || -1, headers: _.mergeWith(potentialLastResult?.headers || {}, _.omit(headers, 'result'), (objValue, srcValue) => { if (Array.isArray(objValue) && Array.isArray(srcValue)) { return [...objValue, ...srcValue]; } return undefined; }), contentType: headers['Content-Type'] || headers['content-type'], }; if (crawlOpts?.proxyUrl) { sideLoadOpts.proxyOrigin[finalURL.origin] = crawlOpts.proxyUrl; } if (headers.result?.code && [301, 302, 307, 308].includes(headers.result.code)) { const location = headers.Location || headers.location; if (location) { finalURL = new URL(location, finalURL); } } } const lastHeaders = curlResult.headers[curlResult.headers.length - 1]; const contentType = (lastHeaders['Content-Type'] || lastHeaders['content-type'])?.toLowerCase() || (curlResult.data?.type) || 'application/octet-stream'; const contentDisposition = lastHeaders['Content-Disposition'] || lastHeaders['content-disposition']; const fileName = contentDisposition?.match(/filename="([^"]+)"/i)?.[1] || finalURL.pathname.split('/').pop(); if (sideLoadOpts.impersonate[finalURL.href] && (curlResult.data?.size)) { sideLoadOpts.impersonate[finalURL.href].body = curlResult.data; } // This should keep the file from being garbage collected and deleted until this asyncContext/request is done. this.lifeCycleTrack.set(this.asyncLocalContext.ctx, curlResult.data); return { finalURL, sideLoadOpts, chain: curlResult.headers, status: curlResult.statusCode, statusText: curlResult.statusText, headers: lastHeaders, contentType, contentDisposition, fileName, file: curlResult.data }; } digestCurlCode(code: CurlCode, msg: string) { switch (code) { // 400 User errors case CurlCode.CURLE_COULDNT_RESOLVE_HOST: { return new AssertionFailureError(msg); } // Maybe retry but dont retry with curl again case CurlCode.CURLE_OPERATION_TIMEDOUT: case CurlCode.CURLE_UNSUPPORTED_PROTOCOL: case CurlCode.CURLE_PEER_FAILED_VERIFICATION: { return new ServiceBadApproachError(msg); } // Retryable errors case CurlCode.CURLE_PROXY: case CurlCode.CURLE_REMOTE_ACCESS_DENIED: case CurlCode.CURLE_SEND_ERROR: case CurlCode.CURLE_RECV_ERROR: case CurlCode.CURLE_GOT_NOTHING: case CurlCode.CURLE_SSL_CONNECT_ERROR: case CurlCode.CURLE_QUIC_CONNECT_ERROR: case CurlCode.CURLE_COULDNT_RESOLVE_PROXY: case CurlCode.CURLE_COULDNT_CONNECT: case CurlCode.CURLE_PARTIAL_FILE: { return new ServiceBadAttemptError(msg); } default: { return undefined; } } } }