chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
+297
@@ -0,0 +1,297 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
import { spawnSync } from 'node:child_process'
|
||||
import path from 'node:path'
|
||||
import { fileURLToPath } from 'node:url'
|
||||
|
||||
const DEFAULT_MAX_PAGES = 8
|
||||
const DEFAULT_MAX_DEPTH = 2
|
||||
const DEFAULT_SAME_DOMAIN_LIMIT = 5
|
||||
const DEFAULT_MAX_SEARCH_QUERIES = 3
|
||||
const DEFAULT_FETCH_MAX_TEXT_CHARS = 6_000
|
||||
const DEFAULT_FETCH_MAX_LINKS = 150
|
||||
const STRONG_MATCH_SCORE = 8
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
urls: [],
|
||||
query: '',
|
||||
maxPages: DEFAULT_MAX_PAGES,
|
||||
maxDepth: DEFAULT_MAX_DEPTH,
|
||||
sameDomainLimit: DEFAULT_SAME_DOMAIN_LIMIT,
|
||||
maxSearchQueries: DEFAULT_MAX_SEARCH_QUERIES
|
||||
}
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i]
|
||||
const next = argv[i + 1]
|
||||
|
||||
if ((arg === '--url' || arg === '--start-url') && next) {
|
||||
args.urls.push(next)
|
||||
i += 1
|
||||
} else if (arg === '--urls' && next) {
|
||||
args.urls.push(
|
||||
...next
|
||||
.split(',')
|
||||
.map((url) => url.trim())
|
||||
.filter(Boolean)
|
||||
)
|
||||
i += 1
|
||||
} else if (arg === '--query' && next) {
|
||||
args.query = next
|
||||
i += 1
|
||||
} else if (arg === '--max-pages' && next) {
|
||||
args.maxPages = Number(next)
|
||||
i += 1
|
||||
} else if (arg === '--max-depth' && next) {
|
||||
args.maxDepth = Number(next)
|
||||
i += 1
|
||||
} else if (arg === '--same-domain-limit' && next) {
|
||||
args.sameDomainLimit = Number(next)
|
||||
i += 1
|
||||
} else if (arg === '--max-search-queries' && next) {
|
||||
args.maxSearchQueries = Number(next)
|
||||
i += 1
|
||||
}
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
function normalizeUrl(url) {
|
||||
const parsedUrl = new URL(url)
|
||||
|
||||
parsedUrl.hash = ''
|
||||
|
||||
return parsedUrl.toString()
|
||||
}
|
||||
|
||||
function getDomain(url) {
|
||||
return new URL(url).hostname.toLowerCase()
|
||||
}
|
||||
|
||||
function getQueryTerms(query) {
|
||||
return query
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/i)
|
||||
.map((term) => term.trim())
|
||||
.filter((term) => term.length >= 2)
|
||||
}
|
||||
|
||||
function scoreLink(link, query, sourceUrl = '') {
|
||||
const primaryHaystack = `${link.url} ${link.text || ''}`.toLowerCase()
|
||||
const contextHaystack = `${link.context || ''}`.toLowerCase()
|
||||
const sourceDomain = sourceUrl ? getDomain(sourceUrl) : ''
|
||||
const linkDomain = getDomain(link.url)
|
||||
let score = 0
|
||||
|
||||
for (const term of getQueryTerms(query)) {
|
||||
if (primaryHaystack.includes(term)) {
|
||||
score += 4
|
||||
}
|
||||
|
||||
if (contextHaystack.includes(term)) {
|
||||
score += 1
|
||||
}
|
||||
}
|
||||
|
||||
if (sourceDomain && sourceDomain === linkDomain && score > 0) {
|
||||
score += 2
|
||||
}
|
||||
|
||||
return score
|
||||
}
|
||||
|
||||
function runFetchPage(url, query) {
|
||||
const scriptPath = path.join(
|
||||
path.dirname(fileURLToPath(import.meta.url)),
|
||||
'fetch-page.mjs'
|
||||
)
|
||||
const result = spawnSync(process.execPath, [
|
||||
scriptPath,
|
||||
'--url',
|
||||
url,
|
||||
'--query',
|
||||
query,
|
||||
'--mode',
|
||||
'summary',
|
||||
'--max-text-chars',
|
||||
String(DEFAULT_FETCH_MAX_TEXT_CHARS),
|
||||
'--max-links',
|
||||
String(DEFAULT_FETCH_MAX_LINKS)
|
||||
], {
|
||||
encoding: 'utf8',
|
||||
maxBuffer: 8 * 1_024 * 1_024
|
||||
})
|
||||
const output = result.stdout || result.stderr || '{}'
|
||||
|
||||
try {
|
||||
return JSON.parse(output)
|
||||
} catch {
|
||||
return {
|
||||
ok: false,
|
||||
url,
|
||||
error: output.trim() || `fetch-page exited with ${result.status}`
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function enqueueRelevantLinks(queue, page, query, depth, args, visited) {
|
||||
if (depth >= args.maxDepth || !Array.isArray(page.links)) {
|
||||
return
|
||||
}
|
||||
|
||||
const scoredLinks = page.links
|
||||
.map((link) => ({
|
||||
...link,
|
||||
score: scoreLink(link, query, page.url)
|
||||
}))
|
||||
.filter((link) => link.score > 0)
|
||||
.sort((firstLink, secondLink) => secondLink.score - firstLink.score)
|
||||
|
||||
for (const link of scoredLinks) {
|
||||
try {
|
||||
const normalizedUrl = normalizeUrl(link.url)
|
||||
|
||||
if (visited.has(normalizedUrl)) {
|
||||
continue
|
||||
}
|
||||
|
||||
queue.push({
|
||||
url: normalizedUrl,
|
||||
depth: depth + 1,
|
||||
via: page.url,
|
||||
linkText: link.text,
|
||||
linkScore: link.score
|
||||
})
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function hasRelevantLinks(page, query) {
|
||||
return (
|
||||
Array.isArray(page.links) &&
|
||||
page.links.some((link) => scoreLink(link, query, page.url) > 0)
|
||||
)
|
||||
}
|
||||
|
||||
function shouldStopEarly(page, query, depth, maxDepth) {
|
||||
return (
|
||||
page.ok === true &&
|
||||
Array.isArray(page.snippets) &&
|
||||
page.snippets.length > 0 &&
|
||||
Number(page.score || 0) >= STRONG_MATCH_SCORE &&
|
||||
(depth >= maxDepth || !hasRelevantLinks(page, query))
|
||||
)
|
||||
}
|
||||
|
||||
async function crawl(args) {
|
||||
if (args.urls.length === 0) {
|
||||
throw new Error('Missing --url or --urls. Use web search first, then pass promising URLs here.')
|
||||
}
|
||||
|
||||
if (!args.query) {
|
||||
throw new Error('Missing --query')
|
||||
}
|
||||
|
||||
const visited = new Set()
|
||||
const perDomainCounts = new Map()
|
||||
const queue = args.urls.map((url) => ({
|
||||
url: normalizeUrl(url),
|
||||
depth: 0,
|
||||
via: null,
|
||||
linkText: '',
|
||||
linkScore: 0
|
||||
}))
|
||||
const pages = []
|
||||
const matches = []
|
||||
let stopReason = 'limit_reached'
|
||||
|
||||
while (queue.length > 0 && pages.length < args.maxPages) {
|
||||
queue.sort((firstItem, secondItem) => secondItem.linkScore - firstItem.linkScore)
|
||||
|
||||
const item = queue.shift()
|
||||
|
||||
if (!item || visited.has(item.url)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const domain = getDomain(item.url)
|
||||
const domainCount = perDomainCounts.get(domain) || 0
|
||||
|
||||
if (domainCount >= args.sameDomainLimit) {
|
||||
continue
|
||||
}
|
||||
|
||||
visited.add(item.url)
|
||||
perDomainCounts.set(domain, domainCount + 1)
|
||||
|
||||
const page = runFetchPage(item.url, args.query)
|
||||
const pageSummary = {
|
||||
url: page.url || item.url,
|
||||
status: page.status || null,
|
||||
ok: page.ok === true,
|
||||
title: page.title || '',
|
||||
depth: item.depth,
|
||||
via: item.via,
|
||||
linkText: item.linkText,
|
||||
score: Number(page.score || 0),
|
||||
textLength: Number(page.textLength || 0),
|
||||
chunk: page.chunk || null,
|
||||
rawTruncated: page.rawTruncated === true,
|
||||
snippets: Array.isArray(page.snippets) ? page.snippets : [],
|
||||
error: page.error || null
|
||||
}
|
||||
|
||||
pages.push(pageSummary)
|
||||
|
||||
if (pageSummary.snippets.length > 0) {
|
||||
matches.push(pageSummary)
|
||||
}
|
||||
|
||||
if (shouldStopEarly(page, args.query, item.depth, args.maxDepth)) {
|
||||
stopReason = 'strong_match_found'
|
||||
break
|
||||
}
|
||||
|
||||
enqueueRelevantLinks(queue, page, args.query, item.depth, args, visited)
|
||||
}
|
||||
|
||||
if (queue.length === 0 && stopReason === 'limit_reached') {
|
||||
stopReason = 'queue_exhausted'
|
||||
}
|
||||
|
||||
return {
|
||||
query: args.query,
|
||||
limits: {
|
||||
maxPages: args.maxPages,
|
||||
maxDepth: args.maxDepth,
|
||||
sameDomainLimit: args.sameDomainLimit,
|
||||
maxSearchQueries: args.maxSearchQueries
|
||||
},
|
||||
stopReason,
|
||||
pagesChecked: pages.length,
|
||||
pages,
|
||||
matches
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await crawl(parseArgs(process.argv.slice(2)))
|
||||
|
||||
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`)
|
||||
} catch (error) {
|
||||
process.stdout.write(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}\n`
|
||||
)
|
||||
process.exitCode = 1
|
||||
}
|
||||
+406
@@ -0,0 +1,406 @@
|
||||
#!/usr/bin/env node
|
||||
|
||||
const DEFAULT_TIMEOUT_MS = 15_000
|
||||
const DEFAULT_MAX_RAW_CHARS = 500_000
|
||||
const DEFAULT_MAX_TEXT_CHARS = 8_000
|
||||
const DEFAULT_TEXT_PREVIEW_CHARS = 1_500
|
||||
const DEFAULT_MAX_LINKS = 150
|
||||
const DEFAULT_MODE = 'summary'
|
||||
const USER_AGENT =
|
||||
'Mozilla/5.0 (Windows NT 11.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/149.0.7815.2 Safari/537.36'
|
||||
const REMOVED_BLOCK_TAGS = [
|
||||
'script',
|
||||
'style',
|
||||
'noscript',
|
||||
'svg',
|
||||
'canvas',
|
||||
'iframe',
|
||||
'form',
|
||||
'select',
|
||||
'button',
|
||||
'nav',
|
||||
'footer',
|
||||
'aside'
|
||||
]
|
||||
|
||||
function parseArgs(argv) {
|
||||
const args = {
|
||||
url: '',
|
||||
query: '',
|
||||
mode: DEFAULT_MODE,
|
||||
includeText: false,
|
||||
offset: 0,
|
||||
timeoutMs: DEFAULT_TIMEOUT_MS,
|
||||
maxRawChars: DEFAULT_MAX_RAW_CHARS,
|
||||
maxTextChars: DEFAULT_MAX_TEXT_CHARS,
|
||||
maxLinks: DEFAULT_MAX_LINKS
|
||||
}
|
||||
|
||||
for (let i = 0; i < argv.length; i += 1) {
|
||||
const arg = argv[i]
|
||||
const next = argv[i + 1]
|
||||
|
||||
if (arg === '--url' && next) {
|
||||
args.url = next
|
||||
i += 1
|
||||
} else if (arg === '--query' && next) {
|
||||
args.query = next
|
||||
i += 1
|
||||
} else if (arg === '--mode' && next) {
|
||||
args.mode = next === 'full' ? 'full' : DEFAULT_MODE
|
||||
i += 1
|
||||
} else if (arg === '--include-text') {
|
||||
args.includeText = next !== 'false'
|
||||
|
||||
if (next === 'true' || next === 'false') {
|
||||
i += 1
|
||||
}
|
||||
} else if (arg === '--offset' && next) {
|
||||
args.offset = Number(next)
|
||||
i += 1
|
||||
} else if (arg === '--timeout-ms' && next) {
|
||||
args.timeoutMs = Number(next)
|
||||
i += 1
|
||||
} else if (arg === '--max-raw-chars' && next) {
|
||||
args.maxRawChars = Number(next)
|
||||
i += 1
|
||||
} else if (arg === '--max-text-chars' && next) {
|
||||
args.maxTextChars = Number(next)
|
||||
i += 1
|
||||
} else if (arg === '--max-links' && next) {
|
||||
args.maxLinks = Number(next)
|
||||
i += 1
|
||||
}
|
||||
}
|
||||
|
||||
return args
|
||||
}
|
||||
|
||||
function getPositiveInteger(value, fallback) {
|
||||
return Number.isFinite(value) && value > 0 ? Math.floor(value) : fallback
|
||||
}
|
||||
|
||||
function decodeHtmlEntities(value) {
|
||||
return value
|
||||
.replace(/ /gi, ' ')
|
||||
.replace(/&/gi, '&')
|
||||
.replace(/"/gi, '"')
|
||||
.replace(/'/g, "'")
|
||||
.replace(/</gi, '<')
|
||||
.replace(/>/gi, '>')
|
||||
.replace(/&#(\d+);/g, (_match, code) => {
|
||||
const charCode = Number(code)
|
||||
|
||||
return Number.isFinite(charCode) ? String.fromCodePoint(charCode) : ''
|
||||
})
|
||||
.replace(/&#x([0-9a-f]+);/gi, (_match, code) => {
|
||||
const charCode = Number.parseInt(code, 16)
|
||||
|
||||
return Number.isFinite(charCode) ? String.fromCodePoint(charCode) : ''
|
||||
})
|
||||
}
|
||||
|
||||
function normalizeWhitespace(value) {
|
||||
return value.replace(/\s+/g, ' ').trim()
|
||||
}
|
||||
|
||||
function removeBlockTag(html, tagName) {
|
||||
return html.replace(
|
||||
new RegExp(`<${tagName}\\b[^>]*>[\\s\\S]*?<\\/${tagName}>`, 'gi'),
|
||||
' '
|
||||
)
|
||||
}
|
||||
|
||||
function stripHtml(html) {
|
||||
const cleanedHtml = REMOVED_BLOCK_TAGS.reduce(
|
||||
(result, tagName) => removeBlockTag(result, tagName),
|
||||
html
|
||||
)
|
||||
|
||||
return normalizeWhitespace(
|
||||
decodeHtmlEntities(
|
||||
cleanedHtml
|
||||
.replace(/<!--[\s\S]*?-->/g, ' ')
|
||||
.replace(/<\/(p|div|section|article|header|footer|main|li|tr|h[1-6])>/gi, '\n')
|
||||
.replace(/<[^>]+>/g, ' ')
|
||||
)
|
||||
)
|
||||
}
|
||||
|
||||
function extractTitle(html) {
|
||||
const titleMatch = html.match(/<title\b[^>]*>([\s\S]*?)<\/title>/i)
|
||||
|
||||
return titleMatch ? normalizeWhitespace(stripHtml(titleMatch[1])) : ''
|
||||
}
|
||||
|
||||
function normalizeUrl(url) {
|
||||
const parsedUrl = new URL(url)
|
||||
|
||||
parsedUrl.hash = ''
|
||||
|
||||
return parsedUrl.toString()
|
||||
}
|
||||
|
||||
function extractLinks(html, baseUrl, maxLinks) {
|
||||
const links = []
|
||||
const seen = new Map()
|
||||
const linkPattern = /<a\b[^>]*href\s*=\s*(?:"([^"]*)"|'([^']*)'|([^\s>]+))[^>]*>([\s\S]*?)<\/a>/gi
|
||||
|
||||
for (const match of html.matchAll(linkPattern)) {
|
||||
const rawHref = match[1] || match[2] || match[3] || ''
|
||||
const label = normalizeWhitespace(stripHtml(match[4] || ''))
|
||||
const contextStart = Math.max(0, match.index - 800)
|
||||
const contextEnd = Math.min(html.length, match.index + match[0].length + 800)
|
||||
const context = normalizeWhitespace(
|
||||
stripHtml(html.slice(contextStart, contextEnd))
|
||||
)
|
||||
|
||||
if (!rawHref || rawHref.startsWith('#')) {
|
||||
continue
|
||||
}
|
||||
|
||||
try {
|
||||
const resolvedUrl = normalizeUrl(new URL(rawHref, baseUrl).toString())
|
||||
const parsedUrl = new URL(resolvedUrl)
|
||||
|
||||
if (!['http:', 'https:'].includes(parsedUrl.protocol)) {
|
||||
continue
|
||||
}
|
||||
|
||||
const existingIndex = seen.get(resolvedUrl)
|
||||
|
||||
if (typeof existingIndex === 'number') {
|
||||
const existingLink = links[existingIndex]
|
||||
|
||||
existingLink.text = normalizeWhitespace(
|
||||
`${existingLink.text} ${label}`
|
||||
).slice(0, 180)
|
||||
existingLink.context = normalizeWhitespace(
|
||||
`${existingLink.context} ${context}`
|
||||
).slice(0, 600)
|
||||
continue
|
||||
}
|
||||
|
||||
seen.set(resolvedUrl, links.length)
|
||||
links.push({
|
||||
url: resolvedUrl,
|
||||
text: label.slice(0, 180),
|
||||
context: context.slice(0, 600)
|
||||
})
|
||||
|
||||
if (links.length >= maxLinks) {
|
||||
break
|
||||
}
|
||||
} catch {
|
||||
continue
|
||||
}
|
||||
}
|
||||
|
||||
return links
|
||||
}
|
||||
|
||||
function getQueryTerms(query) {
|
||||
return query
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9]+/i)
|
||||
.map((term) => term.trim())
|
||||
.filter((term) => term.length >= 2)
|
||||
}
|
||||
|
||||
function escapeRegExp(value) {
|
||||
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')
|
||||
}
|
||||
|
||||
function findSnippets(text, query, maxSnippets = 8) {
|
||||
const terms = getQueryTerms(query)
|
||||
|
||||
if (terms.length === 0) {
|
||||
return []
|
||||
}
|
||||
|
||||
const snippets = []
|
||||
const seen = new Set()
|
||||
|
||||
for (const term of terms) {
|
||||
const pattern = new RegExp(escapeRegExp(term), 'ig')
|
||||
let match = pattern.exec(text)
|
||||
|
||||
while (match && snippets.length < maxSnippets) {
|
||||
const start = Math.max(0, match.index - 180)
|
||||
const end = Math.min(text.length, match.index + term.length + 180)
|
||||
const snippet = normalizeWhitespace(text.slice(start, end))
|
||||
const key = snippet.toLowerCase()
|
||||
|
||||
if (!seen.has(key)) {
|
||||
seen.add(key)
|
||||
snippets.push(snippet)
|
||||
}
|
||||
|
||||
match = pattern.exec(text)
|
||||
}
|
||||
|
||||
if (snippets.length >= maxSnippets) {
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
return snippets
|
||||
}
|
||||
|
||||
function getTextChunk(text, offset, maxTextChars) {
|
||||
const safeOffset = Math.min(
|
||||
Math.max(0, getPositiveInteger(offset, 0)),
|
||||
text.length
|
||||
)
|
||||
const safeMaxTextChars = getPositiveInteger(
|
||||
maxTextChars,
|
||||
DEFAULT_MAX_TEXT_CHARS
|
||||
)
|
||||
const textChunk = text.slice(safeOffset, safeOffset + safeMaxTextChars)
|
||||
const nextOffset = safeOffset + textChunk.length
|
||||
const hasMore = nextOffset < text.length
|
||||
|
||||
return {
|
||||
text: textChunk,
|
||||
offset: safeOffset,
|
||||
chars: textChunk.length,
|
||||
hasMore,
|
||||
nextOffset: hasMore ? nextOffset : null
|
||||
}
|
||||
}
|
||||
|
||||
function scoreText(text, query) {
|
||||
const lowerText = text.toLowerCase()
|
||||
|
||||
return getQueryTerms(query).reduce((score, term) => {
|
||||
const matches = lowerText.match(new RegExp(escapeRegExp(term), 'g'))
|
||||
|
||||
return score + (matches ? matches.length : 0)
|
||||
}, 0)
|
||||
}
|
||||
|
||||
async function readResponseText(response, maxRawChars) {
|
||||
const safeMaxRawChars = getPositiveInteger(
|
||||
maxRawChars,
|
||||
DEFAULT_MAX_RAW_CHARS
|
||||
)
|
||||
|
||||
if (!response.body || typeof response.body.getReader !== 'function') {
|
||||
const raw = await response.text()
|
||||
|
||||
return {
|
||||
raw: raw.slice(0, safeMaxRawChars),
|
||||
rawTruncated: raw.length > safeMaxRawChars
|
||||
}
|
||||
}
|
||||
|
||||
const reader = response.body.getReader()
|
||||
const decoder = new TextDecoder()
|
||||
let raw = ''
|
||||
let rawTruncated = false
|
||||
|
||||
while (raw.length < safeMaxRawChars) {
|
||||
const { done, value } = await reader.read()
|
||||
|
||||
if (done) {
|
||||
break
|
||||
}
|
||||
|
||||
raw += decoder.decode(value, { stream: true })
|
||||
|
||||
if (raw.length >= safeMaxRawChars) {
|
||||
raw = raw.slice(0, safeMaxRawChars)
|
||||
rawTruncated = true
|
||||
await reader.cancel()
|
||||
break
|
||||
}
|
||||
}
|
||||
|
||||
if (!rawTruncated) {
|
||||
raw += decoder.decode()
|
||||
}
|
||||
|
||||
return {
|
||||
raw,
|
||||
rawTruncated
|
||||
}
|
||||
}
|
||||
|
||||
async function fetchPage(args) {
|
||||
if (!args.url) {
|
||||
throw new Error('Missing --url')
|
||||
}
|
||||
|
||||
const controller = new AbortController()
|
||||
const timeout = setTimeout(() => controller.abort(), args.timeoutMs)
|
||||
|
||||
try {
|
||||
const response = await fetch(args.url, {
|
||||
redirect: 'follow',
|
||||
signal: controller.signal,
|
||||
headers: {
|
||||
'user-agent': USER_AGENT,
|
||||
accept: 'text/html,application/xhtml+xml,text/plain;q=0.9,*/*;q=0.5'
|
||||
}
|
||||
})
|
||||
const finalUrl = normalizeUrl(response.url || args.url)
|
||||
const contentType = response.headers.get('content-type') || ''
|
||||
const { raw, rawTruncated } = await readResponseText(
|
||||
response,
|
||||
args.maxRawChars
|
||||
)
|
||||
const isHtml = /html|xml/i.test(contentType) || /<html|<body|<a\s/i.test(raw)
|
||||
const readableText = isHtml ? stripHtml(raw) : normalizeWhitespace(raw)
|
||||
const chunk = getTextChunk(readableText, args.offset, args.maxTextChars)
|
||||
const title = isHtml ? extractTitle(raw) : ''
|
||||
const links = isHtml ? extractLinks(raw, finalUrl, args.maxLinks) : []
|
||||
const includeText = args.mode === 'full' || args.includeText
|
||||
const result = {
|
||||
ok: response.ok,
|
||||
status: response.status,
|
||||
url: finalUrl,
|
||||
contentType,
|
||||
title,
|
||||
mode: args.mode,
|
||||
rawTruncated,
|
||||
textLength: readableText.length,
|
||||
chunk: {
|
||||
offset: chunk.offset,
|
||||
chars: chunk.chars,
|
||||
hasMore: chunk.hasMore,
|
||||
nextOffset: chunk.nextOffset
|
||||
},
|
||||
score: args.query ? scoreText(`${title} ${readableText}`, args.query) : 0,
|
||||
snippets: args.query ? findSnippets(readableText, args.query) : [],
|
||||
links
|
||||
}
|
||||
|
||||
if (includeText) {
|
||||
result.text = chunk.text
|
||||
} else {
|
||||
result.textPreview = chunk.text.slice(0, DEFAULT_TEXT_PREVIEW_CHARS)
|
||||
}
|
||||
|
||||
return result
|
||||
} finally {
|
||||
clearTimeout(timeout)
|
||||
}
|
||||
}
|
||||
|
||||
try {
|
||||
const result = await fetchPage(parseArgs(process.argv.slice(2)))
|
||||
|
||||
process.stdout.write(`${JSON.stringify(result, null, 2)}\n`)
|
||||
} catch (error) {
|
||||
process.stdout.write(
|
||||
`${JSON.stringify(
|
||||
{
|
||||
ok: false,
|
||||
error: error instanceof Error ? error.message : String(error)
|
||||
},
|
||||
null,
|
||||
2
|
||||
)}\n`
|
||||
)
|
||||
process.exitCode = 1
|
||||
}
|
||||
Reference in New Issue
Block a user