wehub-resource-sync
9b395f5cc3
E2E Headed Chrome / e2e-headed (macos-15) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (ubuntu-latest) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (windows-latest) (push) Has been cancelled
CI / build (macos-latest) (push) Has been cancelled
CI / build (ubuntu-latest) (push) Has been cancelled
CI / build (windows-latest) (push) Has been cancelled
CI / unit-test (push) Has been cancelled
CI / bun-test (push) Has been cancelled
CI / adapter-test (push) Has been cancelled
CI / smoke-test (macos-latest) (push) Has been cancelled
CI / smoke-test (ubuntu-latest) (push) Has been cancelled
Security Audit / audit (push) Has been cancelled
Build Chrome Extension / build (push) Has been cancelled
Trigger Website Rebuild (Docs Updated) / dispatch (push) Has been cancelled
381 lines
14 KiB
JavaScript
381 lines
14 KiB
JavaScript
import { CliError } from '@jackwener/opencli/errors';
|
|
export const BLOOMBERG_FEEDS = {
|
|
main: 'https://feeds.bloomberg.com/news.rss',
|
|
markets: 'https://feeds.bloomberg.com/markets/news.rss',
|
|
economics: 'https://feeds.bloomberg.com/economics/news.rss',
|
|
industries: 'https://feeds.bloomberg.com/industries/news.rss',
|
|
tech: 'https://feeds.bloomberg.com/technology/news.rss',
|
|
politics: 'https://feeds.bloomberg.com/politics/news.rss',
|
|
opinions: 'https://feeds.bloomberg.com/bview/news.rss',
|
|
green: 'https://feeds.bloomberg.com/green/news.rss',
|
|
crypto: 'https://feeds.bloomberg.com/crypto/news.rss',
|
|
pursuits: 'https://feeds.bloomberg.com/pursuits/news.rss',
|
|
};
|
|
// Note: the Businessweek RSS feed (feeds.bloomberg.com/businessweek/news.rss) is now served
|
|
// empty by Bloomberg, so the `businessweek` command reads the section page instead (see
|
|
// businessweek.js). Other sections still publish working RSS feeds.
|
|
const DEFAULT_USER_AGENT = 'Mozilla/5.0 (compatible; opencli)';
|
|
// Bloomberg's edge occasionally serves a transient empty/non-OK RSS response under load; a
|
|
// couple of quick retries turn those intermittent misses into a successful fetch instead of a
|
|
// hard NOT_FOUND. A feed that is genuinely empty still surfaces NOT_FOUND after the retries.
|
|
export async function fetchBloombergFeed(name, limit = 1) {
|
|
const feedUrl = BLOOMBERG_FEEDS[name];
|
|
if (!feedUrl) {
|
|
throw new CliError('ARGUMENT', `Unknown Bloomberg feed: ${name}`);
|
|
}
|
|
let lastError;
|
|
for (let attempt = 0; attempt < 3; attempt += 1) {
|
|
if (attempt > 0) {
|
|
await new Promise((resolve) => setTimeout(resolve, 400 * attempt));
|
|
}
|
|
const resp = await fetch(feedUrl, {
|
|
headers: { 'User-Agent': DEFAULT_USER_AGENT },
|
|
});
|
|
if (!resp.ok) {
|
|
lastError = new CliError('FETCH_ERROR', `Bloomberg RSS HTTP ${resp.status}`, 'Bloomberg may be temporarily unavailable; try again later.');
|
|
continue;
|
|
}
|
|
const xml = await resp.text();
|
|
const items = parseBloombergRss(xml);
|
|
if (items.length) {
|
|
const count = Math.max(1, Math.min(Number(limit) || 1, 20));
|
|
return items.slice(0, count);
|
|
}
|
|
lastError = new CliError('NOT_FOUND', 'Bloomberg RSS feed returned no items', 'Bloomberg may have changed the feed format.');
|
|
}
|
|
throw lastError;
|
|
}
|
|
export function parseBloombergRss(xml) {
|
|
const items = [];
|
|
const itemRegex = /<item\b[^>]*>([\s\S]*?)<\/item>/gi;
|
|
let match;
|
|
while ((match = itemRegex.exec(xml))) {
|
|
const block = match[1];
|
|
const title = extractTagText(block, 'title');
|
|
const summary = extractTagText(block, 'description');
|
|
const link = extractTagText(block, 'link') || extractTagText(block, 'guid');
|
|
const mediaLinks = extractMediaLinksFromRssItem(block);
|
|
if (!title || !link)
|
|
continue;
|
|
items.push({
|
|
title,
|
|
summary,
|
|
link,
|
|
mediaLinks,
|
|
});
|
|
}
|
|
return items;
|
|
}
|
|
export function normalizeBloombergLink(input) {
|
|
const raw = String(input || '').trim();
|
|
if (!raw) {
|
|
throw new CliError('ARGUMENT', 'A Bloomberg link is required');
|
|
}
|
|
if (raw.startsWith('/'))
|
|
return `https://www.bloomberg.com${raw}`;
|
|
return raw;
|
|
}
|
|
export function validateBloombergLink(input) {
|
|
const normalized = normalizeBloombergLink(input);
|
|
let url;
|
|
try {
|
|
url = new URL(normalized);
|
|
}
|
|
catch {
|
|
throw new CliError('ARGUMENT', `Invalid Bloomberg link: ${input}`, 'Pass a full https://www.bloomberg.com/... URL or a relative Bloomberg path.');
|
|
}
|
|
if (!/(?:\.|^)bloomberg\.com$/i.test(url.hostname)) {
|
|
throw new CliError('ARGUMENT', `Expected a bloomberg.com link, got: ${url.hostname}`, 'Pass a Bloomberg article URL from bloomberg.com.');
|
|
}
|
|
return url.toString();
|
|
}
|
|
export function renderStoryBody(body) {
|
|
const blocks = Array.isArray(body?.content) ? body.content : [];
|
|
const parts = blocks
|
|
.map((block) => renderBlock(block, 0))
|
|
.map((part) => normalizeBlockText(part))
|
|
.filter(Boolean);
|
|
return parts.join('\n\n').replace(/\n{3,}/g, '\n\n').trim();
|
|
}
|
|
export function extractStoryMediaLinks(story) {
|
|
const urls = new Set();
|
|
collectMediaUrls(story?.ledeImageUrl, urls);
|
|
collectMediaUrls(story?.socialImageUrl, urls);
|
|
collectMediaUrls(story?.lede, urls);
|
|
collectMediaUrls(story?.imageAttachments, urls);
|
|
collectMediaUrls(story?.videoAttachments, urls);
|
|
const mediaBlocks = Array.isArray(story?.body?.content)
|
|
? story.body.content.filter((block) => block?.type === 'media')
|
|
: [];
|
|
collectMediaUrls(mediaBlocks, urls);
|
|
return [...urls];
|
|
}
|
|
function renderBlock(block, depth) {
|
|
if (!block || typeof block !== 'object')
|
|
return '';
|
|
switch (block.type) {
|
|
case 'paragraph':
|
|
return renderInlineNodes(block.content || []);
|
|
case 'heading': {
|
|
const text = renderInlineNodes(block.content || []);
|
|
if (!text)
|
|
return '';
|
|
const level = Number(block.data?.level ?? block.data?.weight ?? 2);
|
|
const prefix = level <= 1 ? '# ' : level === 2 ? '## ' : '### ';
|
|
return `${prefix}${text}`;
|
|
}
|
|
case 'blockquote': {
|
|
const text = renderInlineNodes(block.content || []);
|
|
if (!text)
|
|
return '';
|
|
return text.split('\n').map((line) => line ? `> ${line}` : '>').join('\n');
|
|
}
|
|
case 'list':
|
|
return renderListBlock(block, depth);
|
|
case 'tabularData':
|
|
return renderTabularDataBlock(block);
|
|
case 'media':
|
|
return renderMediaBlock(block);
|
|
case 'inline-newsletter':
|
|
case 'newsletter':
|
|
case 'ad':
|
|
return '';
|
|
default: {
|
|
if (Array.isArray(block.content) && block.content.length > 0) {
|
|
const inlineText = renderInlineNodes(block.content);
|
|
if (inlineText)
|
|
return inlineText;
|
|
const nested = block.content.map((child) => renderBlock(child, depth + 1)).filter(Boolean);
|
|
if (nested.length)
|
|
return nested.join('\n');
|
|
}
|
|
return extractGenericText(block);
|
|
}
|
|
}
|
|
}
|
|
function renderInlineNodes(nodes) {
|
|
return nodes.map((node) => renderInlineNode(node)).join('');
|
|
}
|
|
function renderInlineNode(node) {
|
|
if (node == null)
|
|
return '';
|
|
if (typeof node === 'string')
|
|
return decodeXmlEntities(node);
|
|
switch (node.type) {
|
|
case 'text':
|
|
return decodeXmlEntities(node.value || '');
|
|
case 'linebreak':
|
|
return '\n';
|
|
case 'link':
|
|
case 'entity':
|
|
case 'strong':
|
|
case 'emphasis':
|
|
case 'italic':
|
|
case 'underline':
|
|
case 'span':
|
|
if (Array.isArray(node.content) && node.content.length > 0) {
|
|
return renderInlineNodes(node.content);
|
|
}
|
|
return decodeXmlEntities(node.value || '');
|
|
default:
|
|
if (Array.isArray(node.content) && node.content.length > 0) {
|
|
return renderInlineNodes(node.content);
|
|
}
|
|
if (typeof node.value === 'string')
|
|
return decodeXmlEntities(node.value);
|
|
return '';
|
|
}
|
|
}
|
|
function renderListBlock(block, depth) {
|
|
const items = Array.isArray(block.content) ? block.content : [];
|
|
if (!items.length)
|
|
return '';
|
|
const listStyle = String(block.subType || block.data?.style || block.data?.listType || '');
|
|
const ordered = /\bordered\b|\bnumber(?:ed)?\b/i.test(listStyle);
|
|
let index = 1;
|
|
return items
|
|
.map((item) => {
|
|
const prefix = ordered ? `${index++}. ` : '- ';
|
|
return renderListItem(item, prefix, depth);
|
|
})
|
|
.filter(Boolean)
|
|
.join('\n');
|
|
}
|
|
function renderListItem(item, prefix, depth) {
|
|
const indent = ' '.repeat(depth);
|
|
const body = normalizeBlockText(renderListItemBody(item, depth + 1));
|
|
if (!body)
|
|
return '';
|
|
const lines = body.split('\n');
|
|
const head = `${indent}${prefix}${lines[0]}`;
|
|
if (lines.length === 1)
|
|
return head;
|
|
const continuationIndent = `${indent}${' '.repeat(prefix.length)}`;
|
|
const tail = lines.slice(1).map((line) => `${continuationIndent}${line}`).join('\n');
|
|
return `${head}\n${tail}`;
|
|
}
|
|
function renderListItemBody(item, depth) {
|
|
if (!item || typeof item !== 'object')
|
|
return '';
|
|
if (item.type === 'list-item' && Array.isArray(item.content)) {
|
|
const parts = item.content
|
|
.map((child) => child?.type === 'paragraph'
|
|
? renderInlineNodes(child.content || [])
|
|
: renderBlock(child, depth))
|
|
.map((part) => normalizeBlockText(part))
|
|
.filter(Boolean);
|
|
return parts.join('\n');
|
|
}
|
|
return renderBlock(item, depth);
|
|
}
|
|
function renderTabularDataBlock(block) {
|
|
const rows = block?.data?.rows ?? block?.data?.table?.rows ?? block?.content;
|
|
if (!Array.isArray(rows) || !rows.length) {
|
|
return extractGenericText(block.data || block.content || block);
|
|
}
|
|
const lines = rows
|
|
.map((row) => extractGenericText(row))
|
|
.map((line) => normalizeBlockText(line))
|
|
.filter(Boolean);
|
|
return lines.join('\n');
|
|
}
|
|
function renderMediaBlock(block) {
|
|
const candidates = [
|
|
block?.data?.chart?.caption,
|
|
block?.data?.attachment?.caption,
|
|
block?.data?.attachment?.title,
|
|
block?.data?.attachment?.subtitle,
|
|
block?.data?.video?.caption,
|
|
];
|
|
const caption = candidates
|
|
.map((value) => normalizeBlockText(stripHtml(String(value || ''))))
|
|
.find(Boolean);
|
|
return caption || '';
|
|
}
|
|
function extractGenericText(value) {
|
|
const parts = [];
|
|
collectText(value, parts);
|
|
return parts.join(' ').replace(/\s+/g, ' ').trim();
|
|
}
|
|
function collectText(value, out) {
|
|
if (value == null)
|
|
return;
|
|
if (typeof value === 'string') {
|
|
const text = normalizeBlockText(stripHtml(decodeXmlEntities(value)));
|
|
if (text)
|
|
out.push(text);
|
|
return;
|
|
}
|
|
if (Array.isArray(value)) {
|
|
for (const item of value)
|
|
collectText(item, out);
|
|
return;
|
|
}
|
|
if (typeof value === 'object') {
|
|
if (typeof value.value === 'string') {
|
|
const text = normalizeBlockText(stripHtml(decodeXmlEntities(value.value)));
|
|
if (text)
|
|
out.push(text);
|
|
return;
|
|
}
|
|
if (Array.isArray(value.content)) {
|
|
collectText(value.content, out);
|
|
return;
|
|
}
|
|
for (const entry of Object.values(value))
|
|
collectText(entry, out);
|
|
}
|
|
}
|
|
function extractTagText(block, tag) {
|
|
const safeTag = escapeRegExp(tag);
|
|
const match = block.match(new RegExp(`<${safeTag}(?:\\s[^>]*)?>([\\s\\S]*?)<\\/${safeTag}>`, 'i'));
|
|
if (!match)
|
|
return '';
|
|
return normalizeBlockText(stripHtml(decodeXmlEntities(stripCdata(match[1]))));
|
|
}
|
|
function extractMediaLinksFromRssItem(block) {
|
|
const links = new Set();
|
|
const mediaRegex = /<(?:media:content|media:thumbnail|enclosure)\b[^>]*\burl="([^"]+)"[^>]*>/gi;
|
|
let match;
|
|
while ((match = mediaRegex.exec(block))) {
|
|
const url = decodeXmlEntities(match[1] || '').trim();
|
|
if (url)
|
|
links.add(url);
|
|
}
|
|
return [...links];
|
|
}
|
|
function collectMediaUrls(value, out, seen = new WeakSet()) {
|
|
if (value == null)
|
|
return;
|
|
if (typeof value === 'string') {
|
|
const normalized = normalizeMediaUrl(value);
|
|
if (normalized)
|
|
out.add(normalized);
|
|
return;
|
|
}
|
|
if (Array.isArray(value)) {
|
|
for (const item of value)
|
|
collectMediaUrls(item, out, seen);
|
|
return;
|
|
}
|
|
if (typeof value === 'object') {
|
|
if (seen.has(value))
|
|
return;
|
|
seen.add(value);
|
|
for (const key of ['url', 'src', 'fallback', 'poster']) {
|
|
const candidate = value[key];
|
|
if (typeof candidate === 'string') {
|
|
const normalized = normalizeMediaUrl(candidate);
|
|
if (normalized)
|
|
out.add(normalized);
|
|
}
|
|
}
|
|
for (const entry of Object.values(value)) {
|
|
collectMediaUrls(entry, out, seen);
|
|
}
|
|
}
|
|
}
|
|
function normalizeMediaUrl(value) {
|
|
const url = decodeXmlEntities(String(value || '')).trim();
|
|
if (!/^https?:\/\//i.test(url))
|
|
return null;
|
|
if (!looksLikeMediaUrl(url))
|
|
return null;
|
|
return url;
|
|
}
|
|
function looksLikeMediaUrl(url) {
|
|
return /(?:assets\.bwbx\.io|resource\.bloomberg\.com|media\.bloomberg\.com)/i.test(url)
|
|
|| /\.(?:jpg|jpeg|png|webp|gif|svg|mp4|m3u8)(?:[?#].*)?$/i.test(url);
|
|
}
|
|
function stripCdata(value) {
|
|
const match = value.match(/^<!\[CDATA\[([\s\S]*?)\]\]>$/);
|
|
return match ? match[1] : value;
|
|
}
|
|
function stripHtml(value) {
|
|
return String(value || '').replace(/<[^>]+>/g, ' ');
|
|
}
|
|
function decodeXmlEntities(value) {
|
|
return String(value || '')
|
|
.replace(/<!\[CDATA\[([\s\S]*?)\]\]>/g, '$1')
|
|
.replace(/&#(\d+);/g, (_m, code) => String.fromCodePoint(Number(code)))
|
|
.replace(/&#x([0-9a-f]+);/gi, (_m, code) => String.fromCodePoint(parseInt(code, 16)))
|
|
.replace(/&/g, '&')
|
|
.replace(/</g, '<')
|
|
.replace(/>/g, '>')
|
|
.replace(/"/g, '"')
|
|
.replace(/'/g, "'")
|
|
.replace(/'/g, "'")
|
|
.replace(/ /g, ' ');
|
|
}
|
|
function normalizeBlockText(value) {
|
|
return String(value || '')
|
|
.replace(/\r/g, '')
|
|
.replace(/[ \t]+\n/g, '\n')
|
|
.replace(/\n[ \t]+/g, '\n')
|
|
.replace(/[ \t]{2,}/g, ' ')
|
|
.trim();
|
|
}
|
|
function escapeRegExp(value) {
|
|
return value.replace(/[.*+?^${}()|[\]\\]/g, '\\$&');
|
|
}
|