wehub-resource-sync
54 lines
1.7 KiB
JavaScript
54 lines
1.7 KiB
JavaScript
// HTTP transport helpers shared across providers.
|
|
// Files prefixed with _ are never loaded as providers by scan.mjs.
|
|
|
|
const DEFAULT_TIMEOUT_MS = 10_000;
|
|
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; career-ops/1.3)';
|
|
|
|
async function fetchWithTimeout(url, { timeoutMs = DEFAULT_TIMEOUT_MS, headers = {}, method = 'GET', body = null, redirect = 'follow' } = {}) {
|
|
const controller = new AbortController();
|
|
const timer = setTimeout(() => controller.abort(), timeoutMs);
|
|
try {
|
|
const res = await fetch(url, {
|
|
method,
|
|
headers: { 'user-agent': DEFAULT_USER_AGENT, ...headers },
|
|
body,
|
|
redirect,
|
|
signal: controller.signal,
|
|
});
|
|
if (!res.ok) {
|
|
const responseText = await res.text().catch(() => '');
|
|
// WAF/CDN challenge pages (seen live: Workday 429s) carry no actionable
|
|
// text — HTML markup or a generic interstitial message, not worth
|
|
// parsing or displaying. The status code and its standard reason
|
|
// phrase are what a log line needs; the raw body is still attached as
|
|
// err.body for callers that want to inspect it.
|
|
const err = new Error(`HTTP ${res.status}${res.statusText ? ` ${res.statusText}` : ''}`);
|
|
err.status = res.status;
|
|
err.body = responseText;
|
|
err.retryAfter = res.headers.get('retry-after');
|
|
throw err;
|
|
}
|
|
return res;
|
|
} finally {
|
|
clearTimeout(timer);
|
|
}
|
|
}
|
|
|
|
export async function fetchJson(url, opts = {}) {
|
|
const res = await fetchWithTimeout(url, opts);
|
|
return await res.json();
|
|
}
|
|
|
|
export async function fetchText(url, opts = {}) {
|
|
const res = await fetchWithTimeout(url, opts);
|
|
return await res.text();
|
|
}
|
|
|
|
export function makeHttpCtx() {
|
|
return {
|
|
transport: 'http',
|
|
fetchJson,
|
|
fetchText,
|
|
};
|
|
}
|