wehub-resource-sync
9b395f5cc3
E2E Headed Chrome / e2e-headed (macos-15) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (ubuntu-latest) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (windows-latest) (push) Has been cancelled
CI / build (macos-latest) (push) Has been cancelled
CI / build (ubuntu-latest) (push) Has been cancelled
CI / build (windows-latest) (push) Has been cancelled
CI / unit-test (push) Has been cancelled
CI / bun-test (push) Has been cancelled
CI / adapter-test (push) Has been cancelled
CI / smoke-test (macos-latest) (push) Has been cancelled
CI / smoke-test (ubuntu-latest) (push) Has been cancelled
Security Audit / audit (push) Has been cancelled
Build Chrome Extension / build (push) Has been cancelled
Trigger Website Rebuild (Docs Updated) / dispatch (push) Has been cancelled
372 lines
14 KiB
JavaScript
372 lines
14 KiB
JavaScript
/**
|
||
* WeChat article download — export WeChat Official Account articles to Markdown.
|
||
*
|
||
* Ported from jackwener/wechat-article-to-markdown (JS version) to OpenCLI adapter.
|
||
*
|
||
* Usage:
|
||
* opencli weixin download --url "https://mp.weixin.qq.com/s/xxx" --output ./weixin
|
||
*/
|
||
import { cli, Strategy } from '@jackwener/opencli/registry';
|
||
import { downloadArticle } from '@jackwener/opencli/download/article-download';
|
||
// ============================================================
|
||
// URL Normalization
|
||
// ============================================================
|
||
/**
|
||
* Normalize a pasted WeChat article URL.
|
||
*/
|
||
// Wrapping quote characters to strip from a pasted URL. Covers ASCII plus
|
||
// CJK typographic / smart quotes, which are common when users copy URLs from
|
||
// Chinese-language environments (WeChat itself, macOS smart-quote
|
||
// substitution, Word / Pages, …).
|
||
//
|
||
// Pairs:
|
||
// " " ASCII straight double
|
||
// ' ' ASCII straight single
|
||
// “ ” curly double (U+201C / U+201D)
|
||
// ‘ ’ curly single (U+2018 / U+2019)
|
||
// 「 」 CJK corner brackets (U+300C / U+300D)
|
||
// 『 』 CJK white corner brackets (U+300E / U+300F)
|
||
// „ ‟ German-style double quotes (U+201E / U+201F)
|
||
// ‹ › single guillemets (U+2039 / U+203A)
|
||
// « » double guillemets (U+00AB / U+00BB)
|
||
const WRAPPING_QUOTE_PAIRS = [
|
||
['"', '"'],
|
||
["'", "'"],
|
||
['“', '”'],
|
||
['‘', '’'],
|
||
['「', '」'],
|
||
['『', '』'],
|
||
['„', '‟'],
|
||
['‹', '›'],
|
||
['«', '»'],
|
||
];
|
||
const LEADING_WRAP_CHARS = new Set(WRAPPING_QUOTE_PAIRS.map(([open]) => open).concat('<'));
|
||
const TRAILING_WRAP_CHARS = new Set(WRAPPING_QUOTE_PAIRS.map(([, close]) => close).concat('>'));
|
||
|
||
function stripBoundaryWrapChars(value) {
|
||
let s = value;
|
||
for (let i = 0; i < 4; i += 1) {
|
||
const before = s;
|
||
for (const [open, close] of WRAPPING_QUOTE_PAIRS) {
|
||
if (s.length >= 2 && s.startsWith(open) && s.endsWith(close)) {
|
||
s = s.slice(open.length, s.length - close.length).trim();
|
||
break;
|
||
}
|
||
}
|
||
while (s && LEADING_WRAP_CHARS.has(s[0])) {
|
||
s = s.slice(1).trimStart();
|
||
}
|
||
while (s && TRAILING_WRAP_CHARS.has(s[s.length - 1])) {
|
||
s = s.slice(0, -1).trimEnd();
|
||
}
|
||
if (s === before)
|
||
break;
|
||
}
|
||
return s;
|
||
}
|
||
|
||
export function normalizeWechatUrl(raw) {
|
||
let s = (raw || '').trim();
|
||
if (!s)
|
||
return s;
|
||
// Strip quote / angle-bracket characters only at the pasted boundary. This
|
||
// handles both paired wrappers ("<url>", "“url”") and common one-sided
|
||
// trailing punctuation ("url”") without touching encoded URL content.
|
||
s = stripBoundaryWrapChars(s);
|
||
// Remove backslash escapes before URL-significant characters
|
||
s = s.replace(/\\+([:/&?=#%])/g, '$1');
|
||
// Decode HTML entities
|
||
s = s.replace(/&/g, '&').replace(/</g, '<').replace(/>/g, '>').replace(/"/g, '"');
|
||
// Allow bare hostnames
|
||
if (s.startsWith('mp.weixin.qq.com/') || s.startsWith('//mp.weixin.qq.com/')) {
|
||
s = 'https://' + s.replace(/^\/+/, '');
|
||
}
|
||
// Force https for mp.weixin.qq.com
|
||
try {
|
||
const parsed = new URL(s);
|
||
if (['http:', 'https:'].includes(parsed.protocol) && parsed.hostname.toLowerCase() === 'mp.weixin.qq.com') {
|
||
parsed.protocol = 'https:';
|
||
s = parsed.toString();
|
||
}
|
||
}
|
||
catch {
|
||
// Ignore parse errors
|
||
}
|
||
return s;
|
||
}
|
||
/**
|
||
* Format a WeChat article timestamp as a UTC+8 datetime string.
|
||
* Accepts either Unix seconds or milliseconds.
|
||
*/
|
||
export function formatWechatTimestamp(rawTimestamp) {
|
||
const ts = Number.parseInt(rawTimestamp, 10);
|
||
if (!Number.isFinite(ts) || ts <= 0)
|
||
return '';
|
||
const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000;
|
||
const d = new Date(timestampMs);
|
||
const pad = (n) => String(n).padStart(2, '0');
|
||
const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
|
||
return (`${utc8.getUTCFullYear()}-` +
|
||
`${pad(utc8.getUTCMonth() + 1)}-` +
|
||
`${pad(utc8.getUTCDate())} ` +
|
||
`${pad(utc8.getUTCHours())}:` +
|
||
`${pad(utc8.getUTCMinutes())}:` +
|
||
`${pad(utc8.getUTCSeconds())}`);
|
||
}
|
||
/**
|
||
* Extract the raw create_time value from supported WeChat inline script formats.
|
||
*/
|
||
export function extractWechatCreateTimeValue(htmlStr) {
|
||
const jsDecodeMatch = htmlStr.match(/create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/);
|
||
if (jsDecodeMatch)
|
||
return jsDecodeMatch[1];
|
||
const directValueMatch = htmlStr.match(/create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/);
|
||
if (!directValueMatch)
|
||
return '';
|
||
return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || '';
|
||
}
|
||
/**
|
||
* Extract the publish time from DOM text first, then fall back to numeric create_time values.
|
||
*/
|
||
export function extractWechatPublishTime(publishTimeText, htmlStr) {
|
||
const normalizedPublishTime = (publishTimeText || '').trim();
|
||
if (normalizedPublishTime)
|
||
return normalizedPublishTime;
|
||
const rawCreateTime = extractWechatCreateTimeValue(htmlStr);
|
||
if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime))
|
||
return '';
|
||
return formatWechatTimestamp(rawCreateTime);
|
||
}
|
||
/**
|
||
* Detect WeChat anti-bot / verification gate pages before we try to parse the article.
|
||
*/
|
||
export function detectWechatAccessIssue(pageText, htmlStr) {
|
||
const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
|
||
if (/环境异常/.test(normalizedText) &&
|
||
/(完成验证后即可继续访问|去验证)/.test(normalizedText)) {
|
||
return 'environment verification required';
|
||
}
|
||
if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
|
||
return 'environment verification required';
|
||
}
|
||
return '';
|
||
}
|
||
export function pickFirstWechatMetaText(...candidates) {
|
||
for (const candidate of candidates) {
|
||
const normalized = (candidate || '').replace(/\s+/g, ' ').trim();
|
||
if (normalized && normalized !== 'Name cleared')
|
||
return normalized;
|
||
}
|
||
return '';
|
||
}
|
||
/**
|
||
* Build a self-contained helper for execution inside page.evaluate().
|
||
*/
|
||
export function buildExtractWechatPublishTimeJs() {
|
||
return `(${function extractWechatPublishTimeInPage(publishTimeText, htmlStr) {
|
||
function formatWechatTimestamp(rawTimestamp) {
|
||
const ts = Number.parseInt(rawTimestamp, 10);
|
||
if (!Number.isFinite(ts) || ts <= 0)
|
||
return '';
|
||
const timestampMs = rawTimestamp.length === 13 ? ts : ts * 1000;
|
||
const d = new Date(timestampMs);
|
||
const pad = (n) => String(n).padStart(2, '0');
|
||
const utc8 = new Date(d.getTime() + 8 * 3600 * 1000);
|
||
return (`${utc8.getUTCFullYear()}-` +
|
||
`${pad(utc8.getUTCMonth() + 1)}-` +
|
||
`${pad(utc8.getUTCDate())} ` +
|
||
`${pad(utc8.getUTCHours())}:` +
|
||
`${pad(utc8.getUTCMinutes())}:` +
|
||
`${pad(utc8.getUTCSeconds())}`);
|
||
}
|
||
function extractWechatCreateTimeValue(html) {
|
||
const jsDecodeMatch = html.match(/create_time\s*:\s*JsDecode\('([^']+)'\)(?=[\s,;}]|$)/);
|
||
if (jsDecodeMatch)
|
||
return jsDecodeMatch[1];
|
||
const directValueMatch = html.match(/create_time\s*[:=]\s*(?:"([^"]+)"|'([^']+)'|([0-9A-Za-z]+))(?=[\s,;}]|$)/);
|
||
if (!directValueMatch)
|
||
return '';
|
||
return directValueMatch[1] || directValueMatch[2] || directValueMatch[3] || '';
|
||
}
|
||
const normalizedPublishTime = (publishTimeText || '').trim();
|
||
if (normalizedPublishTime)
|
||
return normalizedPublishTime;
|
||
const rawCreateTime = extractWechatCreateTimeValue(htmlStr);
|
||
if (!/^\d{10}$|^\d{13}$/.test(rawCreateTime))
|
||
return '';
|
||
return formatWechatTimestamp(rawCreateTime);
|
||
}.toString()})`;
|
||
}
|
||
/**
|
||
* Build a self-contained access-issue detector for execution inside page.evaluate().
|
||
*/
|
||
export function buildDetectWechatAccessIssueJs() {
|
||
return `(${function detectWechatAccessIssueInPage(pageText, htmlStr) {
|
||
const normalizedText = (pageText || '').replace(/\s+/g, ' ').trim();
|
||
if (/环境异常/.test(normalizedText) &&
|
||
/(完成验证后即可继续访问|去验证)/.test(normalizedText)) {
|
||
return 'environment verification required';
|
||
}
|
||
if (/secitptpage\/verify\.html/.test(htmlStr) || /id=["']js_verify["']/.test(htmlStr)) {
|
||
return 'environment verification required';
|
||
}
|
||
return '';
|
||
}.toString()})`;
|
||
}
|
||
// ============================================================
|
||
// CLI Registration
|
||
// ============================================================
|
||
cli({
|
||
site: 'weixin',
|
||
name: 'download',
|
||
access: 'read',
|
||
description: '下载微信公众号文章为 Markdown 格式',
|
||
domain: 'mp.weixin.qq.com',
|
||
strategy: Strategy.COOKIE,
|
||
args: [
|
||
{ name: 'url', required: true, help: 'WeChat article URL (mp.weixin.qq.com/s/xxx)' },
|
||
{ name: 'output', default: './weixin-articles', help: 'Output directory' },
|
||
{ name: 'download-images', type: 'boolean', default: true, help: 'Download images locally' },
|
||
],
|
||
columns: ['title', 'author', 'publish_time', 'status', 'size', 'saved'],
|
||
func: async (page, kwargs) => {
|
||
const rawUrl = kwargs.url;
|
||
const url = normalizeWechatUrl(rawUrl);
|
||
if (!url.startsWith('https://mp.weixin.qq.com/')) {
|
||
return [{ title: 'Error', author: '-', publish_time: '-', status: 'invalid URL', size: '-', saved: '-' }];
|
||
}
|
||
// Navigate and wait for content to load
|
||
await page.goto(url);
|
||
await page.wait(5);
|
||
// Extract article data in browser context
|
||
const data = await page.evaluate(`
|
||
(() => {
|
||
const result = {
|
||
title: '',
|
||
author: '',
|
||
publishTime: '',
|
||
errorHint: '',
|
||
contentHtml: '',
|
||
codeBlocks: [],
|
||
imageUrls: []
|
||
};
|
||
|
||
const pickFirstText = (...selectors) => {
|
||
for (const selector of selectors) {
|
||
const text = document.querySelector(selector)?.textContent?.replace(/\\s+/g, ' ').trim() || '';
|
||
if (text && text !== 'Name cleared') return text;
|
||
}
|
||
return '';
|
||
};
|
||
|
||
// WeChat has multiple article templates. Newer pages use #js_text_title.
|
||
result.title = pickFirstText(
|
||
'#activity-name',
|
||
'#js_text_title',
|
||
'.rich_media_title',
|
||
);
|
||
|
||
result.author = pickFirstText(
|
||
'#js_name',
|
||
'.wx_follow_nickname',
|
||
'#profileBt .profile_nickname',
|
||
'.rich_media_meta.rich_media_meta_nickname',
|
||
'.rich_media_meta_nickname',
|
||
);
|
||
|
||
// Publish time: prefer the rendered DOM text, then fall back to numeric create_time values.
|
||
const publishTimeEl = document.querySelector('#publish_time');
|
||
const extractWechatPublishTime = ${buildExtractWechatPublishTimeJs()};
|
||
result.publishTime = extractWechatPublishTime(
|
||
publishTimeEl ? publishTimeEl.textContent : '',
|
||
document.documentElement.innerHTML,
|
||
);
|
||
|
||
const detectWechatAccessIssue = ${buildDetectWechatAccessIssueJs()};
|
||
result.errorHint = detectWechatAccessIssue(
|
||
document.body ? document.body.innerText : '',
|
||
document.documentElement.innerHTML,
|
||
);
|
||
if (result.errorHint) return result;
|
||
|
||
// Content processing
|
||
const contentEl = document.querySelector('#js_content');
|
||
if (!contentEl) return result;
|
||
|
||
// Fix lazy-loaded images: data-src -> src
|
||
contentEl.querySelectorAll('img').forEach(img => {
|
||
const dataSrc = img.getAttribute('data-src');
|
||
if (dataSrc) img.setAttribute('src', dataSrc);
|
||
});
|
||
|
||
// Extract code blocks with placeholder replacement
|
||
const codeBlocks = [];
|
||
contentEl.querySelectorAll('.code-snippet__fix').forEach(el => {
|
||
el.querySelectorAll('.code-snippet__line-index').forEach(li => li.remove());
|
||
const pre = el.querySelector('pre[data-lang]');
|
||
const lang = pre ? (pre.getAttribute('data-lang') || '') : '';
|
||
const lines = [];
|
||
el.querySelectorAll('code').forEach(codeTag => {
|
||
const text = codeTag.textContent;
|
||
if (/^[ce]?ounter\\(line/.test(text)) return;
|
||
lines.push(text);
|
||
});
|
||
if (lines.length === 0) lines.push(el.textContent);
|
||
const placeholder = 'CODEBLOCK-PLACEHOLDER-' + codeBlocks.length;
|
||
codeBlocks.push({ lang, code: lines.join('\\n') });
|
||
const p = document.createElement('p');
|
||
p.textContent = placeholder;
|
||
el.replaceWith(p);
|
||
});
|
||
result.codeBlocks = codeBlocks;
|
||
|
||
// Remove noise elements
|
||
['script', 'style', '.qr_code_pc', '.reward_area'].forEach(sel => {
|
||
contentEl.querySelectorAll(sel).forEach(tag => tag.remove());
|
||
});
|
||
|
||
// Collect image URLs (deduplicated)
|
||
const seen = new Set();
|
||
contentEl.querySelectorAll('img[src]').forEach(img => {
|
||
const src = img.getAttribute('src');
|
||
if (src && !seen.has(src)) {
|
||
seen.add(src);
|
||
result.imageUrls.push(src);
|
||
}
|
||
});
|
||
|
||
result.contentHtml = contentEl.innerHTML;
|
||
return result;
|
||
})()
|
||
`);
|
||
if (data?.errorHint === 'environment verification required') {
|
||
return [{
|
||
title: 'Error',
|
||
author: '-',
|
||
publish_time: '-',
|
||
status: 'failed — verification required in WeChat browser page',
|
||
size: '-',
|
||
saved: '-',
|
||
}];
|
||
}
|
||
return downloadArticle({
|
||
title: data?.title || '',
|
||
author: data?.author,
|
||
publishTime: data?.publishTime,
|
||
sourceUrl: url,
|
||
contentHtml: data?.contentHtml || '',
|
||
codeBlocks: data?.codeBlocks,
|
||
imageUrls: data?.imageUrls,
|
||
}, {
|
||
output: kwargs.output,
|
||
downloadImages: kwargs['download-images'],
|
||
imageHeaders: { Referer: 'https://mp.weixin.qq.com/' },
|
||
frontmatterLabels: { author: '公众号' },
|
||
detectImageExt: (url) => {
|
||
const m = url.match(/wx_fmt=(\w+)/) || url.match(/\.(\w{3,4})(?:\?|$)/);
|
||
return m ? m[1] : 'png';
|
||
},
|
||
});
|
||
},
|
||
});
|