wehub-resource-sync
1028 lines
37 KiB
TypeScript
1028 lines
37 KiB
TypeScript
import { Also, Coercible, ParamValidationError, Prop, RPC_CALL_ENVIRONMENT } from 'civkit/civ-rpc';
|
|
import { FancyFile } from 'civkit/fancy-file';
|
|
import { Cookie, parseString as parseSetCookieString } from 'set-cookie-parser';
|
|
import { Context } from '../services/registry';
|
|
import { TurnDownTweakableOptions } from './turndown-tweakable-options';
|
|
import type { PageSnapshot } from '../services/puppeteer';
|
|
import { PseudoBoolean, PseudoBooleanLoose } from '../lib/pseudo-boolean';
|
|
import _ from 'lodash';
|
|
|
|
export enum CONTENT_FORMAT {
|
|
CONTENT = 'content',
|
|
MARKDOWN = 'markdown',
|
|
HTML = 'html',
|
|
TEXT = 'text',
|
|
PAGESHOT = 'pageshot',
|
|
SCREENSHOT = 'screenshot',
|
|
VLM = 'vlm',
|
|
READER_LM = 'readerlm-v2',
|
|
FRONTMATTER = 'frontmatter',
|
|
}
|
|
|
|
export enum ENGINE_TYPE {
|
|
AUTO = 'auto',
|
|
BROWSER = 'browser',
|
|
CURL = 'curl',
|
|
CF_BROWSER_RENDERING = 'cf-browser-rendering',
|
|
}
|
|
|
|
export enum RESPOND_TIMING {
|
|
HTML = 'html',
|
|
VISIBLE_CONTENT = 'visible-content',
|
|
MUTATION_IDLE = 'mutation-idle',
|
|
RESOURCE_IDLE = 'resource-idle',
|
|
MEDIA_IDLE = 'media-idle',
|
|
NETWORK_IDLE = 'network-idle',
|
|
}
|
|
|
|
export enum CHUNKING_STRATEGY {
|
|
ENABLED = 'true',
|
|
ENABLED1 = 'h1',
|
|
ENABLED2 = 'h2',
|
|
ENABLED3 = 'h3',
|
|
ENABLED4 = 'h4',
|
|
ENABLED5 = 'h5',
|
|
STRUCTURED = 'structured',
|
|
STRUCTURED1 = 's1',
|
|
STRUCTURED2 = 's2',
|
|
STRUCTURED3 = 's3',
|
|
STRUCTURED4 = 's4',
|
|
STRUCTURED5 = 's5',
|
|
}
|
|
|
|
|
|
const CONTENT_FORMAT_VALUES = new Set<string>(Object.values(CONTENT_FORMAT));
|
|
|
|
export const IMAGE_RETENTION_MODES = ['none', 'all', 'alt', 'all_p', 'alt_p'] as const;
|
|
const IMAGE_RETENTION_MODE_VALUES = new Set<string>(IMAGE_RETENTION_MODES);
|
|
export const LINK_RETENTION_MODES = ['none', 'all', 'text', 'gpt-oss'] as const;
|
|
const LINK_RETENTION_MODE_VALUES = new Set<string>(LINK_RETENTION_MODES);
|
|
export const MEDIA_RETENTION_MODES = ['none', 'text', 'link', 'image', 'html'] as const;
|
|
const MEDIA_RETENTION_MODE_VALUES = new Set<string>(MEDIA_RETENTION_MODES);
|
|
export const BASE_URL_MODES = ['initial', 'final'] as const;
|
|
const BASE_URL_MODE_VALUES = new Set<string>(BASE_URL_MODES);
|
|
export const PRESET_NAMES = ['reader', 'index', 'research', 'agent', 'spider'] as const;
|
|
const PRESET_NAME_VALUES = new Set<string>(PRESET_NAMES);
|
|
|
|
// Fields that still equal their system default are eligible for preset override.
|
|
const PRESET_OPTIONS: Record<string, Partial<CrawlerOptions>> = {
|
|
reader: {
|
|
retainImages: 'all',
|
|
retainMedia: 'html',
|
|
retainLinks: 'all',
|
|
respondWith: CONTENT_FORMAT.FRONTMATTER,
|
|
detachInvisibles: true,
|
|
removeOverlay: true,
|
|
markdown: {
|
|
linkStyle: 'referenced',
|
|
linkReferenceStyle: 'full',
|
|
},
|
|
base: 'final',
|
|
},
|
|
index: {
|
|
retainImages: 'alt',
|
|
retainMedia: 'none',
|
|
retainLinks: 'text',
|
|
respondWith: CONTENT_FORMAT.FRONTMATTER,
|
|
markdownChunking: CHUNKING_STRATEGY.STRUCTURED3,
|
|
base: 'final',
|
|
},
|
|
research: {
|
|
retainImages: 'all',
|
|
retainMedia: 'link',
|
|
retainLinks: 'all',
|
|
respondWith: `${CONTENT_FORMAT.MARKDOWN}+${CONTENT_FORMAT.FRONTMATTER}`,
|
|
markdownChunking: CHUNKING_STRATEGY.ENABLED3,
|
|
base: 'final',
|
|
},
|
|
agent: {
|
|
retainImages: 'alt',
|
|
retainMedia: 'link',
|
|
retainLinks: 'all',
|
|
respondWith: CONTENT_FORMAT.FRONTMATTER,
|
|
markdownChunking: CHUNKING_STRATEGY.ENABLED3,
|
|
base: 'final',
|
|
},
|
|
spider: {
|
|
retainImages: 'all',
|
|
retainMedia: 'link',
|
|
retainLinks: 'all',
|
|
respondWith: `${CONTENT_FORMAT.MARKDOWN}+${CONTENT_FORMAT.FRONTMATTER}`,
|
|
markdownChunking: CHUNKING_STRATEGY.ENABLED3,
|
|
withLinksSummary: 'all',
|
|
base: 'final',
|
|
},
|
|
};
|
|
|
|
class Viewport extends Coercible {
|
|
@Prop({
|
|
default: 1280
|
|
})
|
|
width!: number;
|
|
@Prop({
|
|
default: 1280
|
|
})
|
|
height!: number;
|
|
@Prop()
|
|
deviceScaleFactor?: number;
|
|
@Prop()
|
|
isMobile?: boolean;
|
|
@Prop()
|
|
isLandscape?: boolean;
|
|
@Prop()
|
|
hasTouch?: boolean;
|
|
}
|
|
|
|
@Also({
|
|
openapi: {
|
|
operation: {
|
|
parameters: {
|
|
'Accept': {
|
|
description: `Specifies your preference for the response format.\n\n` +
|
|
`Supported formats: \n` +
|
|
`- text/event-stream\n` +
|
|
`- application/json or text/json\n` +
|
|
`- text/plain`
|
|
,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Cache-Tolerance': {
|
|
description: `Sets internal cache tolerance in seconds if this header is specified with a integer.`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-No-Cache': {
|
|
description: `Ignores internal cache if this header is specified with a value.\n\nEquivalent to X-Cache-Tolerance: 0`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Respond-With': {
|
|
description: `Specifies the (non-default) form of the crawled data you prefer.\n\n` +
|
|
`Supported formats: \n` +
|
|
`- markdown\n` +
|
|
`- html\n` +
|
|
`- text\n` +
|
|
`- pageshot\n` +
|
|
`- screenshot\n` +
|
|
`- content\n` +
|
|
`- frontmatter\n` +
|
|
`- any combination of the above\n` +
|
|
`- readerlm-v2\n` +
|
|
`- vlm\n\n` +
|
|
`Default: content\n`
|
|
,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Wait-For-Selector': {
|
|
description: `Specifies a CSS selector to wait for the appearance of such an element before returning.\n\n` +
|
|
'Example: `X-Wait-For-Selector: .content-block`\n'
|
|
,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Target-Selector': {
|
|
description: `Specifies a CSS selector for return target instead of the full html.\n\n` +
|
|
'Implies `X-Wait-For-Selector: (same selector)`'
|
|
,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Remove-Selector': {
|
|
description: `Specifies a CSS selector to remove elements from the full html.\n\n` +
|
|
'Example `X-Remove-Selector: nav`'
|
|
,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Keep-Img-Data-Url': {
|
|
description: `Keep data-url as it instead of transforming them to object-url. (Only applicable when targeting markdown format)\n\n` +
|
|
'Example `X-Keep-Img-Data-Url: true`'
|
|
,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Proxy-Url': {
|
|
description: `Specifies your custom proxy if you prefer to use one.\n\n` +
|
|
`Supported protocols: \n` +
|
|
`- http\n` +
|
|
`- https\n` +
|
|
`- socks4\n` +
|
|
`- socks5\n\n` +
|
|
`For authentication, https://user:pass@host:port`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Proxy': {
|
|
description: `Use a proxy server provided by us.\n\nOptionally specify two-letter country code.`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Robots-Txt': {
|
|
description: `Load and conform to the respective robot.txt on the target origin.\n\nOptionally specify a bot UA to check against.\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'DNT': {
|
|
description: `When set to 1, prevent the result of this request to be cached in the system.\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Set-Cookie': {
|
|
description: `Sets cookie(s) to the headless browser for your request. \n\n` +
|
|
`Syntax is the same with standard Set-Cookie`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-With-Generated-Alt': {
|
|
description: `Enable automatic alt-text generating for images without an meaningful alt-text.\n\n` +
|
|
`Note: Does not work when \`X-Respond-With\` is specified`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-With-Images-Summary': {
|
|
description: `Enable dedicated summary section for images on the page.`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-With-links-Summary': {
|
|
description: `Enable dedicated summary section for hyper links on the page.`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Retain-Links': {
|
|
description: `Link retention modes.\n\n` +
|
|
`Supported modes: \n` +
|
|
`- all: all links\n` +
|
|
`- none: no links\n` +
|
|
`- text: only link text\n` +
|
|
`- gpt-oss: gpt-oss link citation format \`【{id}†.*】\`\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Retain-Images': {
|
|
description: `Image retention modes.\n\n` +
|
|
`Supported modes: \n` +
|
|
`- all: all images\n` +
|
|
`- none: no images\n` +
|
|
`- alt: only alt text\n` +
|
|
`- all_p: all images and with generated alt text\n` +
|
|
`- alt_p: only alt text and with generated alt\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Retain-Media': {
|
|
description: `Media retention modes for video, audio, and embedded video iframes.\n\n` +
|
|
`Supported modes: \n` +
|
|
`- none: no media\n` +
|
|
`- text: plain label, e.g. \`Video 1\`\n` +
|
|
`- link: markdown link, e.g. \`[Video 1](url)\` (default)\n` +
|
|
`- image: markdown image syntax, e.g. \`\`\n` +
|
|
`- html: original HTML with irrelevant attributes (class, id, style, data-*, aria-*) stripped\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Preset': {
|
|
description: `Apply a preset configuration for common scenarios.\n\n` +
|
|
`Supported presets:\n` +
|
|
`- reader: optimized for display to human users\n` +
|
|
`- index: optimized for semantic indexing\n` +
|
|
`- research: optimized for academic/research AI agents\n` +
|
|
`- agent: optimized for day-to-day AI agents\n` +
|
|
`- spider: optimized for recursive site crawling\n\n` +
|
|
`Preset values are applied only to options not explicitly set by the caller.\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-With-Iframe': {
|
|
description: `Enable filling iframe contents into main. (violates standards)`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-With-Shadow-Dom': {
|
|
description: `Enable filling shadow dom contents into main. (violates standards)`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-User-Agent': {
|
|
description: `Override User-Agent.`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Timeout': {
|
|
description: `Specify timeout in seconds. Max 180.`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Locale': {
|
|
description: 'Specify browser locale for the page.',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Referer': {
|
|
description: 'Specify referer for the page.',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Token-Budget': {
|
|
description: 'Specify a budget in tokens.\n\nIf the resulting token cost exceeds the budget, the request is rejected.\n\nNote this parameter is ignored for the search endpoint.',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Max-Tokens': {
|
|
description: 'Trim the response content at a maximum number of tokens.',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Assert-Status-Code': {
|
|
description: 'Assert the HTTP status code of the crawled page. If the actual status code does not match the asserted one, the request is rejected with 422 Unprocessable Entity.',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Respond-Timing': {
|
|
description: `Explicitly specify the respond timing. One of the following:\n\n` +
|
|
`- html: directly return unrendered HTML\n` +
|
|
`- visible-content: return immediately when any content becomes available\n` +
|
|
`- mutation-idle: wait for DOM mutations to settle and remain unchanged for at least 0.2s\n` +
|
|
`- resource-idle: wait for no additional resources that would affect page logic and content has SUCCEEDED loading in 0.5s\n` +
|
|
`- media-idle: wait for no additional resources, including media resources, has SUCCEEDED loading in 0.5s\n` +
|
|
`- network-idle: wait for full load of webpage, also known as networkidle0.\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Engine': {
|
|
description: 'Specify the engine to use for crawling.\n\nSupported: browser, direct, cf-browser-rendering',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Base': {
|
|
description: 'Select base modes of relative URLs.\n\nSupported: initial, final',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Remove-Overlay': {
|
|
description: 'Specify whether to remove overlay elements from the page.',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Detach-Invisibles': {
|
|
description: 'Temporarily detach elements with computed `display:none` before snapshotting, then restore them.\n\n' +
|
|
'Requires the browser engine and disables caching.',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Markdown-Chunking': {
|
|
description: `Opt-in markdown chunking.\n\nSupported values: \n${Object.values(CHUNKING_STRATEGY).map(x => `- ${x}`).join('\n')}\n\nNote if you are expecting text return, the chunking is done by injecting character \\u001D\n\n`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Md-Heading-Style': {
|
|
description: 'Heading style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: setext, atx',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Md-Hr': {
|
|
description: 'Hr text of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Md-Bullet-List-Marker': {
|
|
description: 'Bullet list marker of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: -, +, *',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Md-Em-Delimiter': {
|
|
description: 'Em delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: _, *',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Md-Strong-Delimiter': {
|
|
description: 'Strong delimiter of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: **, __',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Md-Link-Style': {
|
|
description: 'Link style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: inlined, referenced, discarded',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Md-Link-Reference-Style': {
|
|
description: 'Link reference style of the generated markdown.\n\nThis is an option passed through to [Turndown](https://github.com/mixmark-io/turndown?tab=readme-ov-file#options).\n\nSupported: full, collapsed, shortcut, discarded',
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
'X-Page': {
|
|
description: `Steer which page to return for uploaded files (PDF, DOC, XLS, PPT, etc.).\n\n` +
|
|
`1-indexed; ignored when no file/PDF is uploaded.`,
|
|
in: 'header',
|
|
schema: { type: 'string' }
|
|
},
|
|
}
|
|
}
|
|
}
|
|
})
|
|
export class CrawlerOptions extends Coercible {
|
|
@Prop()
|
|
url?: string;
|
|
|
|
@Prop()
|
|
html?: string;
|
|
|
|
@Prop({
|
|
type: BASE_URL_MODE_VALUES,
|
|
default: 'initial',
|
|
})
|
|
base?: typeof BASE_URL_MODES[number];
|
|
|
|
@Prop({
|
|
desc: 'Base64 encoded PDF.',
|
|
type: [FancyFile, String]
|
|
})
|
|
pdf?: FancyFile | string;
|
|
|
|
@Prop({
|
|
desc: 'Base64 encoded file.',
|
|
type: [FancyFile, String]
|
|
})
|
|
file?: FancyFile | string;
|
|
|
|
@Prop({
|
|
desc: 'Steer which page to return for uploaded files (1-indexed). Ignored when no file/PDF is uploaded.',
|
|
validate: (v: number) => Number.isInteger(v) && v >= 1,
|
|
})
|
|
page?: number;
|
|
|
|
@Prop({
|
|
default: CONTENT_FORMAT.CONTENT,
|
|
type: [CONTENT_FORMAT, String]
|
|
})
|
|
respondWith!: string;
|
|
|
|
@Prop({
|
|
type: PseudoBoolean,
|
|
})
|
|
withGeneratedAlt?: boolean;
|
|
|
|
@Prop({ default: 'all', type: IMAGE_RETENTION_MODE_VALUES })
|
|
retainImages?: typeof IMAGE_RETENTION_MODES[number];
|
|
|
|
@Prop({ default: 'link', type: MEDIA_RETENTION_MODE_VALUES })
|
|
retainMedia?: typeof MEDIA_RETENTION_MODES[number];
|
|
|
|
@Prop({ type: PRESET_NAME_VALUES })
|
|
preset?: typeof PRESET_NAMES[number];
|
|
|
|
@Prop({ default: 'all', type: LINK_RETENTION_MODE_VALUES })
|
|
retainLinks?: typeof LINK_RETENTION_MODES[number];
|
|
|
|
@Prop({
|
|
type: [PseudoBoolean, String],
|
|
})
|
|
withLinksSummary?: boolean | string;
|
|
|
|
@Prop({
|
|
type: PseudoBooleanLoose,
|
|
})
|
|
withImagesSummary?: boolean;
|
|
|
|
@Prop({
|
|
type: PseudoBooleanLoose,
|
|
})
|
|
noCache?: boolean;
|
|
|
|
@Prop({
|
|
type: PseudoBooleanLoose,
|
|
})
|
|
removeOverlay?: boolean;
|
|
|
|
@Prop({
|
|
type: PseudoBooleanLoose,
|
|
})
|
|
detachInvisibles?: boolean;
|
|
|
|
@Prop({
|
|
type: [PseudoBoolean, String],
|
|
})
|
|
noGfm?: string | boolean;
|
|
|
|
@Prop()
|
|
cacheTolerance?: number;
|
|
|
|
@Prop({ arrayOf: String })
|
|
targetSelector?: string | string[];
|
|
|
|
@Prop({ arrayOf: String })
|
|
waitForSelector?: string | string[];
|
|
|
|
@Prop({ arrayOf: String })
|
|
removeSelector?: string | string[];
|
|
|
|
@Prop({
|
|
type: PseudoBooleanLoose,
|
|
})
|
|
keepImgDataUrl?: boolean;
|
|
|
|
@Prop({
|
|
type: [PseudoBoolean, String],
|
|
})
|
|
withIframe?: boolean | 'quoted';
|
|
|
|
@Prop({
|
|
type: PseudoBooleanLoose
|
|
})
|
|
withShadowDom?: boolean;
|
|
|
|
@Prop({
|
|
arrayOf: [Object, String],
|
|
})
|
|
setCookies?: Cookie[];
|
|
|
|
@Prop()
|
|
proxyUrl?: string;
|
|
|
|
@Prop()
|
|
proxy?: string;
|
|
|
|
@Prop()
|
|
userAgent?: string;
|
|
|
|
@Prop({
|
|
type: [ENGINE_TYPE, String],
|
|
})
|
|
engine?: string;
|
|
|
|
@Prop({
|
|
arrayOf: String,
|
|
})
|
|
injectPageScript?: string[];
|
|
|
|
@Prop({
|
|
arrayOf: String,
|
|
})
|
|
injectFrameScript?: string[];
|
|
|
|
@Prop({
|
|
validate: (v: number) => v > 0 && v <= 180,
|
|
type: Number,
|
|
nullable: true,
|
|
})
|
|
timeout?: number | null;
|
|
|
|
@Prop()
|
|
locale?: string;
|
|
|
|
@Prop()
|
|
referer?: string;
|
|
|
|
@Prop()
|
|
tokenBudget?: number;
|
|
|
|
@Prop()
|
|
assertStatusCode?: number;
|
|
|
|
@Prop()
|
|
viewport?: Viewport;
|
|
|
|
@Prop()
|
|
instruction?: string;
|
|
|
|
@Prop()
|
|
jsonSchema?: object;
|
|
|
|
@Prop()
|
|
robotsTxt?: string;
|
|
|
|
@Prop()
|
|
doNotTrack?: number | null;
|
|
|
|
@Prop()
|
|
markdown?: TurnDownTweakableOptions;
|
|
|
|
@Prop({
|
|
type: RESPOND_TIMING,
|
|
})
|
|
respondTiming?: RESPOND_TIMING;
|
|
|
|
@Prop({
|
|
type: [CHUNKING_STRATEGY, String],
|
|
})
|
|
markdownChunking?: string;
|
|
|
|
@Prop({
|
|
validate: (v: number) => v >= 500,
|
|
})
|
|
maxTokens?: number;
|
|
|
|
@Prop({
|
|
dictOf: String,
|
|
})
|
|
customHeader?: { [k: string]: string; };
|
|
|
|
_hintIps?: string[];
|
|
_hintCountry?: string;
|
|
|
|
static override from(input: any) {
|
|
const ctx = Reflect.get(input, RPC_CALL_ENVIRONMENT) as Context | undefined;
|
|
let instance = super.from(input) as CrawlerOptions;
|
|
const presetHeader = ctx?.get('x-preset');
|
|
if (presetHeader && PRESET_NAME_VALUES.has(presetHeader)) {
|
|
instance.preset ??= presetHeader as any;
|
|
}
|
|
if (instance.preset && PRESET_NAME_VALUES.has(instance.preset)) {
|
|
instance = super.from({
|
|
preset: instance.preset,
|
|
...PRESET_OPTIONS[instance.preset as keyof typeof PRESET_OPTIONS],
|
|
...input
|
|
}) as CrawlerOptions;
|
|
}
|
|
|
|
const customMode = ctx?.get('x-respond-with') || ctx?.get('x-return-format');
|
|
if (customMode) {
|
|
instance.respondWith = customMode;
|
|
}
|
|
if (instance.respondWith) {
|
|
instance.respondWith = instance.respondWith.toLowerCase();
|
|
}
|
|
if (instance.respondWith?.includes('lm')) {
|
|
if (instance.respondWith.includes('content') || instance.respondWith.includes('markdown')) {
|
|
throw new ParamValidationError({
|
|
path: 'respondWith',
|
|
message: `LM formats conflicts with content/markdown.`,
|
|
});
|
|
}
|
|
}
|
|
if (instance.respondWith === CONTENT_FORMAT.FRONTMATTER) {
|
|
instance.respondWith = `${CONTENT_FORMAT.CONTENT}+${CONTENT_FORMAT.FRONTMATTER}`;
|
|
}
|
|
|
|
const locale = ctx?.get('x-locale');
|
|
if (locale) {
|
|
instance.locale = locale;
|
|
}
|
|
|
|
const referer = ctx?.get('x-referer');
|
|
if (referer) {
|
|
instance.referer = referer;
|
|
}
|
|
|
|
const withGeneratedAlt = ctx?.get('x-with-generated-alt');
|
|
if (withGeneratedAlt) {
|
|
instance.withGeneratedAlt = Boolean(withGeneratedAlt);
|
|
}
|
|
const withLinksSummary = ctx?.get('x-with-links-summary');
|
|
if (withLinksSummary) {
|
|
if (withLinksSummary === 'all') {
|
|
instance.withLinksSummary = withLinksSummary;
|
|
} else {
|
|
instance.withLinksSummary = Boolean(withLinksSummary);
|
|
}
|
|
}
|
|
const withImagesSummary = ctx?.get('x-with-images-summary');
|
|
if (withImagesSummary) {
|
|
instance.withImagesSummary = Boolean(withImagesSummary);
|
|
}
|
|
const retainImages = ctx?.get('x-retain-images');
|
|
if (retainImages && IMAGE_RETENTION_MODE_VALUES.has(retainImages)) {
|
|
instance.retainImages = retainImages as any;
|
|
}
|
|
if (instance.withGeneratedAlt) {
|
|
instance.retainImages = 'all_p';
|
|
}
|
|
const retainMedia = ctx?.get('x-retain-media');
|
|
if (retainMedia && MEDIA_RETENTION_MODE_VALUES.has(retainMedia)) {
|
|
instance.retainMedia = retainMedia as any;
|
|
}
|
|
const retainLinks = ctx?.get('x-retain-links');
|
|
if (retainLinks && LINK_RETENTION_MODE_VALUES.has(retainLinks)) {
|
|
instance.retainLinks = retainLinks as any;
|
|
}
|
|
if (instance.retainLinks === 'gpt-oss') {
|
|
instance.withLinksSummary = 'gpt-oss';
|
|
}
|
|
const noCache = ctx?.get('x-no-cache');
|
|
if (noCache) {
|
|
instance.noCache = Boolean(noCache);
|
|
}
|
|
if (instance.noCache && instance.cacheTolerance === undefined) {
|
|
instance.cacheTolerance = 0;
|
|
}
|
|
let cacheTolerance = parseInt(ctx?.get('x-cache-tolerance') || '');
|
|
if (!isNaN(cacheTolerance)) {
|
|
instance.cacheTolerance = cacheTolerance;
|
|
}
|
|
|
|
const noGfm = ctx?.get('x-no-gfm');
|
|
if (noGfm) {
|
|
instance.noGfm = noGfm === 'table' ? noGfm : Boolean(noGfm);
|
|
}
|
|
|
|
let timeoutSeconds = parseInt(ctx?.get('x-timeout') || '');
|
|
if (!isNaN(timeoutSeconds) && timeoutSeconds > 0) {
|
|
instance.timeout = timeoutSeconds <= 180 ? timeoutSeconds : 180;
|
|
} else if (ctx?.get('x-timeout')) {
|
|
instance.timeout = null;
|
|
}
|
|
|
|
const removeSelector = ctx?.get('x-remove-selector')?.split(', ').filter(Boolean);
|
|
instance.removeSelector ??= removeSelector?.length ? removeSelector : undefined;
|
|
const targetSelector = ctx?.get('x-target-selector')?.split(', ').filter(Boolean);
|
|
instance.targetSelector ??= targetSelector?.length ? targetSelector : undefined;
|
|
const waitForSelector = ctx?.get('x-wait-for-selector')?.split(', ').filter(Boolean);
|
|
instance.waitForSelector ??= (waitForSelector?.length ? waitForSelector : undefined) || instance.targetSelector;
|
|
const overrideUserAgent = ctx?.get('x-user-agent') || undefined;
|
|
instance.userAgent ??= overrideUserAgent;
|
|
|
|
const removeOverlay = ctx?.get('x-remove-overlay')?.toLowerCase();
|
|
if (removeOverlay) {
|
|
instance.removeOverlay = ['no', 'none', 'false']?.includes(removeOverlay) ? false : true;
|
|
}
|
|
|
|
const detachInvisibles = ctx?.get('x-detach-invisibles')?.toLowerCase();
|
|
if (detachInvisibles) {
|
|
instance.detachInvisibles ??= ['no', 'none', 'false'].includes(detachInvisibles) ? false : true;
|
|
}
|
|
|
|
const engine = ctx?.get('x-engine');
|
|
if (engine) {
|
|
instance.engine = engine;
|
|
}
|
|
if (instance.engine) {
|
|
instance.engine = instance.engine.toLowerCase();
|
|
}
|
|
if (instance.engine === 'vlm') {
|
|
instance.engine = ENGINE_TYPE.BROWSER;
|
|
instance.respondWith = CONTENT_FORMAT.VLM;
|
|
} else if (instance.engine === 'readerlm-v2') {
|
|
instance.engine = ENGINE_TYPE.AUTO;
|
|
instance.respondWith = CONTENT_FORMAT.READER_LM;
|
|
}
|
|
|
|
const keepImgDataUrl = ctx?.get('x-keep-img-data-url');
|
|
if (keepImgDataUrl) {
|
|
instance.keepImgDataUrl = Boolean(keepImgDataUrl);
|
|
}
|
|
const withIframe = ctx?.get('x-with-iframe');
|
|
if (withIframe) {
|
|
instance.withIframe = withIframe.toLowerCase() === 'quoted' ? 'quoted' : Boolean(withIframe);
|
|
}
|
|
if (instance.withIframe) {
|
|
instance.timeout ??= null;
|
|
}
|
|
const withShadowDom = ctx?.get('x-with-shadow-dom');
|
|
if (withShadowDom) {
|
|
instance.withShadowDom = Boolean(withShadowDom);
|
|
}
|
|
if (instance.withShadowDom) {
|
|
instance.timeout ??= null;
|
|
}
|
|
|
|
const cookies: Cookie[] = [];
|
|
const setCookieHeaders = (ctx?.get('x-set-cookie')?.split(', ') || instance.setCookies as any as string[] || []).filter(Boolean);
|
|
if (Array.isArray(setCookieHeaders)) {
|
|
for (const setCookie of setCookieHeaders) {
|
|
cookies.push({
|
|
...parseSetCookieString(setCookie, { decodeValues: true }),
|
|
});
|
|
}
|
|
} else if (setCookieHeaders && typeof setCookieHeaders === 'string') {
|
|
cookies.push({
|
|
...parseSetCookieString(setCookieHeaders, { decodeValues: true }),
|
|
});
|
|
}
|
|
instance.setCookies = cookies;
|
|
|
|
const proxyUrl = ctx?.get('x-proxy-url');
|
|
instance.proxyUrl ??= proxyUrl || undefined;
|
|
const proxy = ctx?.get('x-proxy');
|
|
instance.proxy ??= proxy || undefined;
|
|
const robotsTxt = ctx?.get('x-robots-txt');
|
|
instance.robotsTxt ??= robotsTxt || undefined;
|
|
|
|
const tokenBudget = ctx?.get('x-token-budget');
|
|
instance.tokenBudget ??= parseInt(tokenBudget || '') || undefined;
|
|
|
|
const maxTokens = ctx?.get('x-max-tokens');
|
|
instance.maxTokens ??= parseInt(maxTokens || '') || undefined;
|
|
|
|
const assertStatusCode = ctx?.get('x-assert-status-code');
|
|
instance.assertStatusCode ??= parseInt(assertStatusCode || '') || undefined;
|
|
|
|
const markdownChunking = ctx?.get('x-markdown-chunking');
|
|
instance.markdownChunking ??= markdownChunking || undefined;
|
|
|
|
const baseMode = ctx?.get('x-base');
|
|
if (baseMode) {
|
|
instance.base = baseMode as any;
|
|
}
|
|
|
|
const dnt = ctx?.get('dnt');
|
|
instance.doNotTrack ??= (parseInt(dnt || '') || null);
|
|
|
|
const respondTiming = ctx?.get('x-respond-timing');
|
|
if (respondTiming) {
|
|
instance.respondTiming ??= respondTiming as RESPOND_TIMING;
|
|
}
|
|
|
|
const pageHeader = ctx?.get('x-page');
|
|
if (pageHeader) {
|
|
const parsedPage = parseInt(pageHeader, 10);
|
|
if (Number.isInteger(parsedPage) && parsedPage >= 1) {
|
|
instance.page ??= parsedPage;
|
|
}
|
|
}
|
|
|
|
if (instance.cacheTolerance) {
|
|
instance.cacheTolerance = instance.cacheTolerance * 1000;
|
|
}
|
|
|
|
if (ctx) {
|
|
const mdOptions = TurnDownTweakableOptions.fromCtx(ctx);
|
|
if (!_.isEmpty(mdOptions)) {
|
|
instance.markdown ??= mdOptions;
|
|
}
|
|
}
|
|
|
|
return super.from(instance) as CrawlerOptions;
|
|
}
|
|
|
|
get presumedRespondTiming() {
|
|
if (this.respondTiming) {
|
|
return this.respondTiming;
|
|
}
|
|
if (this.timeout && this.timeout >= 20) {
|
|
return RESPOND_TIMING.NETWORK_IDLE;
|
|
}
|
|
if (this.withIframe) {
|
|
return RESPOND_TIMING.NETWORK_IDLE;
|
|
}
|
|
if (this.respondWith.includes('shot') || this.respondWith.includes('vlm')) {
|
|
return RESPOND_TIMING.MEDIA_IDLE;
|
|
}
|
|
if (this.__snapshotT0) {
|
|
const elapsed = performance.now() - this.__snapshotT0;
|
|
if (elapsed > 5_000) {
|
|
return RESPOND_TIMING.MUTATION_IDLE;
|
|
}
|
|
}
|
|
|
|
return RESPOND_TIMING.RESOURCE_IDLE;
|
|
}
|
|
|
|
__snapshotT0?: number;
|
|
isSnapshotAcceptableForEarlyResponse(snapshot: PageSnapshot) {
|
|
if (this.waitForSelector?.length) {
|
|
return false;
|
|
}
|
|
this.__snapshotT0 ??= performance.now();
|
|
const presumedTiming = this.presumedRespondTiming;
|
|
if (presumedTiming === RESPOND_TIMING.MEDIA_IDLE && snapshot.lastMediaResourceLoaded && snapshot.lastMutationIdle) {
|
|
const now = Date.now();
|
|
if ((Math.max(snapshot.lastMediaResourceLoaded, snapshot.lastContentResourceLoaded || 0) + 500) < now) {
|
|
return true;
|
|
}
|
|
}
|
|
if ((this.respondWith.includes('vlm') || this.respondWith.includes('pageshot')) && !snapshot.pageshot) {
|
|
return false;
|
|
}
|
|
if ((this.respondWith.includes('vlm') || this.respondWith.includes('screenshot')) && !snapshot.screenshot) {
|
|
return false;
|
|
}
|
|
if (presumedTiming === RESPOND_TIMING.RESOURCE_IDLE && snapshot.lastContentResourceLoaded && snapshot.lastMutationIdle) {
|
|
const now = Date.now();
|
|
if ((snapshot.lastContentResourceLoaded + 500) < now) {
|
|
return true;
|
|
}
|
|
}
|
|
if (this.injectFrameScript?.length || this.injectPageScript?.length) {
|
|
return false;
|
|
}
|
|
if (presumedTiming === RESPOND_TIMING.VISIBLE_CONTENT && snapshot.parsed?.content) {
|
|
return true;
|
|
}
|
|
if (presumedTiming === RESPOND_TIMING.HTML && snapshot.html) {
|
|
return true;
|
|
}
|
|
if (presumedTiming === RESPOND_TIMING.NETWORK_IDLE) {
|
|
return false;
|
|
}
|
|
if (presumedTiming === RESPOND_TIMING.MUTATION_IDLE && snapshot.lastMutationIdle) {
|
|
return true;
|
|
}
|
|
if (this.respondWith.includes('lm')) {
|
|
return false;
|
|
}
|
|
if (this.withIframe) {
|
|
return false;
|
|
}
|
|
|
|
return !snapshot.isIntermediate;
|
|
}
|
|
|
|
isCacheQueryApplicable() {
|
|
if (this.noCache) {
|
|
return false;
|
|
}
|
|
if (this.cacheTolerance === 0) {
|
|
return false;
|
|
}
|
|
if (this.setCookies?.length) {
|
|
return false;
|
|
}
|
|
if (this.injectFrameScript?.length || this.injectPageScript?.length) {
|
|
return false;
|
|
}
|
|
if (this.viewport) {
|
|
return false;
|
|
}
|
|
if (this.instruction) {
|
|
return false;
|
|
}
|
|
if (this.removeOverlay) {
|
|
return false;
|
|
}
|
|
if (this.detachInvisibles) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
isRequestingCompoundContentFormat() {
|
|
if (CONTENT_FORMAT_VALUES.has(this.respondWith)) {
|
|
return false;
|
|
}
|
|
|
|
const respondWith = (this.respondWith || '').replaceAll(/[+, ]?frontmatter[+, ]?/ig, '');
|
|
|
|
return !CONTENT_FORMAT_VALUES.has(respondWith);
|
|
}
|
|
|
|
browserIsNotRequired() {
|
|
if (this.respondTiming && ![RESPOND_TIMING.HTML, RESPOND_TIMING.VISIBLE_CONTENT].includes(this.respondTiming)) {
|
|
return false;
|
|
}
|
|
if (this.respondWith.includes(CONTENT_FORMAT.PAGESHOT) || this.respondWith.includes(CONTENT_FORMAT.SCREENSHOT)) {
|
|
return false;
|
|
}
|
|
if (this.injectFrameScript?.length || this.injectPageScript?.length) {
|
|
return false;
|
|
}
|
|
if (this.waitForSelector?.length) {
|
|
return false;
|
|
}
|
|
if (this.withIframe || this.withShadowDom) {
|
|
return false;
|
|
}
|
|
if (this.viewport) {
|
|
return false;
|
|
}
|
|
if (this.pdf) {
|
|
return false;
|
|
}
|
|
if (this.html) {
|
|
return false;
|
|
}
|
|
if (this.detachInvisibles) {
|
|
return false;
|
|
}
|
|
|
|
return true;
|
|
}
|
|
|
|
readabilityRequired() {
|
|
if (this.presumedRespondTiming === RESPOND_TIMING.VISIBLE_CONTENT) {
|
|
return true;
|
|
}
|
|
if (!this.respondWith || this.respondWith.includes('content')) {
|
|
return true;
|
|
}
|
|
|
|
return false;
|
|
}
|
|
|
|
toJSON() {
|
|
return _.omit(this, 'pdf', 'html', 'file');
|
|
}
|
|
|
|
customizedProps() {
|
|
const defaults = (this.constructor as typeof CrawlerOptions).from({}) as this;
|
|
return _.omitBy(this, (value, key) => {
|
|
if (['pdf', 'file', 'html'].includes(key)) {
|
|
return true;
|
|
}
|
|
return _.isEqual(value, Reflect.get(defaults, key));
|
|
});
|
|
}
|
|
}
|
|
|
|
export class CrawlerOptionsHeaderOnly extends CrawlerOptions {
|
|
static override from(input: any) {
|
|
const instance = super.from({
|
|
[RPC_CALL_ENVIRONMENT]: Reflect.get(input, RPC_CALL_ENVIRONMENT),
|
|
}) as CrawlerOptionsHeaderOnly;
|
|
|
|
return instance;
|
|
}
|
|
}
|