chore: import upstream snapshot with attribution

clis/substack/feed.js

@@ -0,0 +1,16 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { buildSubstackBrowseUrl, loadSubstackFeed } from './utils.js';
+cli({
+    site: 'substack',
+    name: 'feed',
+    access: 'read',
+    description: 'Substack 热门文章 Feed',
+    domain: 'substack.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'category', default: 'all', help: '文章分类: all, tech, business, culture, politics, science, health' },
+        { name: 'limit', type: 'int', default: 20, help: '返回的文章数量' },
+    ],
+    columns: ['rank', 'title', 'author', 'date', 'readTime', 'url'],
+    func: async (page, args) => loadSubstackFeed(page, buildSubstackBrowseUrl(args.category), Number(args.limit) || 20),
+});

clis/substack/publication.js

@@ -0,0 +1,16 @@
+import { cli, Strategy } from '@jackwener/opencli/registry';
+import { loadSubstackArchive } from './utils.js';
+cli({
+    site: 'substack',
+    name: 'publication',
+    access: 'read',
+    description: '获取特定 Substack Newsletter 的最新文章',
+    domain: 'substack.com',
+    strategy: Strategy.COOKIE,
+    args: [
+        { name: 'url', required: true, positional: true, help: 'Newsletter URL（如 https://example.substack.com）' },
+        { name: 'limit', type: 'int', default: 20, help: '返回的文章数量' },
+    ],
+    columns: ['rank', 'title', 'date', 'description', 'url'],
+    func: async (page, args) => loadSubstackArchive(page, args.url.replace(/\/$/, ''), Number(args.limit) || 20),
+});

clis/substack/search.js

@@ -0,0 +1,79 @@
+import { CommandExecutionError } from '@jackwener/opencli/errors';
+import { cli, Strategy } from '@jackwener/opencli/registry';
+function headers() {
+    return {
+        'User-Agent': 'Mozilla/5.0',
+        Accept: 'application/json',
+    };
+}
+function trim(value) {
+    return typeof value === 'string' ? value.replace(/\s+/g, ' ').trim() : '';
+}
+function publicationBaseUrl(publication) {
+    if (publication?.custom_domain)
+        return `https://${publication.custom_domain}`;
+    if (publication?.subdomain)
+        return `https://${publication.subdomain}.substack.com`;
+    return '';
+}
+async function searchPosts(keyword, limit) {
+    const url = new URL('https://substack.com/api/v1/post/search');
+    url.searchParams.set('query', keyword);
+    url.searchParams.set('page', '0');
+    url.searchParams.set('includePlatformResults', 'true');
+    const resp = await fetch(url, { headers: headers() });
+    if (!resp.ok)
+        throw new CommandExecutionError(`Substack post search failed: HTTP ${resp.status}`);
+    const data = await resp.json();
+    const results = Array.isArray(data?.results) ? data.results : [];
+    return results.slice(0, limit).map((item, index) => ({
+        rank: index + 1,
+        title: trim(item?.title),
+        author: trim(item?.publishedBylines?.[0]?.name),
+        date: trim(item?.post_date).split('T')[0] || trim(item?.post_date),
+        description: trim(item?.description || item?.subtitle || item?.truncated_body_text).slice(0, 150),
+        url: trim(item?.canonical_url),
+    }));
+}
+async function searchPublications(keyword, limit) {
+    const url = new URL('https://substack.com/api/v1/profile/search');
+    url.searchParams.set('query', keyword);
+    url.searchParams.set('page', '0');
+    const resp = await fetch(url, { headers: headers() });
+    if (!resp.ok)
+        throw new CommandExecutionError(`Substack publication search failed: HTTP ${resp.status}`);
+    const data = await resp.json();
+    const results = Array.isArray(data?.results) ? data.results : [];
+    return results.slice(0, limit).map((item, index) => {
+        const publication = item?.primaryPublication || item?.publicationUsers?.[0]?.publication || {};
+        return {
+            rank: index + 1,
+            title: trim(publication?.name || item?.name),
+            author: trim(item?.name),
+            date: '',
+            description: trim(publication?.hero_text || item?.bio).slice(0, 150),
+            url: publicationBaseUrl(publication),
+        };
+    });
+}
+cli({
+    site: 'substack',
+    name: 'search',
+    access: 'read',
+    description: '搜索 Substack 文章和 Newsletter',
+    domain: 'substack.com',
+    strategy: Strategy.PUBLIC,
+    browser: false,
+    args: [
+        { name: 'keyword', required: true, positional: true, help: '搜索关键词' },
+        { name: 'type', default: 'posts', choices: ['posts', 'publications'], help: '搜索类型（posts=文章, publications=Newsletter）' },
+        { name: 'limit', type: 'int', default: 20, help: '返回结果数量' },
+    ],
+    columns: ['rank', 'title', 'author', 'date', 'description', 'url'],
+    func: async (args) => {
+        const limit = Math.max(1, Math.min(Number(args.limit) || 20, 50));
+        return args.type === 'publications'
+            ? searchPublications(args.keyword, limit)
+            : searchPosts(args.keyword, limit);
+    },
+});

clis/substack/utils.js

@@ -0,0 +1,136 @@
+import { CommandExecutionError } from '@jackwener/opencli/errors';
+const FEED_POST_LINK_SELECTOR = 'a[href*="/home/post/"], a[href*="/p/"]';
+const ARCHIVE_POST_LINK_SELECTOR = 'a[href*="/p/"]';
+export function buildSubstackBrowseUrl(category) {
+    if (!category || category === 'all')
+        return 'https://substack.com/';
+    const slug = category === 'tech' ? 'technology' : category;
+    return `https://substack.com/browse/${slug}`;
+}
+export async function loadSubstackFeed(page, url, limit) {
+    if (!page)
+        throw new CommandExecutionError('Browser session required for substack feed');
+    await page.goto(url);
+    await page.wait({ selector: FEED_POST_LINK_SELECTOR, timeout: 5 });
+    const data = await page.evaluate(`
+    (async () => {
+      await new Promise((resolve) => setTimeout(resolve, 3000));
+      const limit = ${Math.max(1, Math.min(limit, 50))};
+      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const posts = [];
+      const seen = new Set();
+
+      const allLinks = Array.from(document.querySelectorAll('a')).filter((link) => {
+        const href = link.getAttribute('href') || '';
+        return href.includes('/home/post/') || href.includes('/p/');
+      });
+
+      for (const linkEl of allLinks) {
+        let postUrl = linkEl.getAttribute('href') || '';
+        if (!postUrl) continue;
+        if (!postUrl.startsWith('http')) postUrl = 'https://substack.com' + postUrl;
+        if (seen.has(postUrl)) continue;
+
+        const lines = (linkEl.innerText || '')
+          .split('\\n')
+          .map((line) => normalize(line))
+          .filter(Boolean);
+
+        const readMeta = lines.find((line) => /\\b(read|watch|listen)\\b/i.test(line)) || '';
+        if (!readMeta) continue;
+
+        const date = lines.find((line) => /^[A-Z]{3}\\s+\\d{1,2}$/i.test(line)) || '';
+        const contentLines = lines.filter((line) =>
+          line &&
+          line !== date &&
+          line !== readMeta &&
+          line.toLowerCase() !== 'save' &&
+          line.toLowerCase() !== 'more' &&
+          !/^(sign in|create account|get app)$/i.test(line),
+        );
+
+        const metaParts = readMeta.split('∙').map((part) => normalize(part));
+        const author = metaParts[0] || '';
+        const readTime = metaParts.slice(1).join(' ∙ ') || readMeta;
+        const title = contentLines.length >= 2 ? contentLines[1] : (contentLines[0] || '');
+        const description = contentLines.length >= 3 ? contentLines.slice(2).join(' ') : '';
+        if (!title) continue;
+
+        seen.add(postUrl);
+        posts.push({
+          rank: posts.length + 1,
+          title,
+          author,
+          date,
+          readTime,
+          description: description.slice(0, 150),
+          url: postUrl,
+        });
+
+        if (posts.length >= limit) break;
+      }
+
+      return posts;
+    })()
+  `);
+    return Array.isArray(data) ? data : [];
+}
+export async function loadSubstackArchive(page, baseUrl, limit) {
+    if (!page)
+        throw new CommandExecutionError('Browser session required for substack archive');
+    await page.goto(`${baseUrl}/archive`);
+    await page.wait({ selector: ARCHIVE_POST_LINK_SELECTOR, timeout: 5 });
+    const data = await page.evaluate(`
+    (async () => {
+      await new Promise((resolve) => setTimeout(resolve, 3000));
+      const normalize = (value) => (value || '').replace(/\\s+/g, ' ').trim();
+      const limit = ${Math.max(1, Math.min(limit, 50))};
+      const grouped = new Map();
+
+      for (const link of Array.from(document.querySelectorAll('a[href*="/p/"]'))) {
+        const rawHref = link.getAttribute('href') || '';
+        if (!rawHref || rawHref === '/p/upgrade') continue;
+
+        const url = rawHref.startsWith('http') ? rawHref : ${JSON.stringify(baseUrl)} + rawHref;
+        const text = normalize(link.textContent);
+        if (!text) continue;
+        if (/^(subscribe|paid|home|about|latest|top|discussions)$/i.test(text)) continue;
+        if (/^[\\d,]+$/.test(text)) continue;
+
+        const entry = grouped.get(url) || { texts: new Set(), date: '' };
+        entry.texts.add(text);
+
+        const container = link.closest('article, section, div') || link.parentElement || link;
+        const containerText = normalize(container.textContent);
+        if (!entry.date) {
+          entry.date = containerText.match(/\\b(?:[A-Z]{3}\\s+\\d{1,2}|[A-Z][a-z]{2}\\s+\\d{1,2})\\b/)?.[0] || '';
+        }
+
+        grouped.set(url, entry);
+      }
+
+      const posts = [];
+      for (const [url, entry] of Array.from(grouped.entries())) {
+        const texts = Array.from(entry.texts).map((text) => normalize(text)).filter((text) => text.length > 3).sort((a, b) => a.length - b.length);
+        const title = texts[0] || '';
+        const description = texts.find((text) => text !== title) || '';
+        if (!title) continue;
+        posts.push({
+          rank: posts.length + 1,
+          title,
+          date: entry.date,
+          description: description.slice(0, 150),
+          url,
+        });
+        if (posts.length >= limit) break;
+      }
+
+      return posts;
+    })()
+  `);
+    return Array.isArray(data) ? data : [];
+}
+export const __test__ = {
+    FEED_POST_LINK_SELECTOR,
+    ARCHIVE_POST_LINK_SELECTOR,
+};

clis/substack/utils.test.js

@@ -0,0 +1,44 @@
+import { describe, expect, it, vi } from 'vitest';
+import { __test__, loadSubstackArchive, loadSubstackFeed } from './utils.js';
+function createPageMock(evaluateResult) {
+    return {
+        goto: vi.fn().mockResolvedValue(undefined),
+        evaluate: vi.fn().mockResolvedValue(evaluateResult),
+        snapshot: vi.fn().mockResolvedValue(undefined),
+        click: vi.fn().mockResolvedValue(undefined),
+        typeText: vi.fn().mockResolvedValue(undefined),
+        pressKey: vi.fn().mockResolvedValue(undefined),
+        scrollTo: vi.fn().mockResolvedValue(undefined),
+        getFormState: vi.fn().mockResolvedValue({}),
+        wait: vi.fn().mockResolvedValue(undefined),
+        tabs: vi.fn().mockResolvedValue([]),
+        selectTab: vi.fn().mockResolvedValue(undefined),
+        networkRequests: vi.fn().mockResolvedValue([]),
+        consoleMessages: vi.fn().mockResolvedValue([]),
+        scroll: vi.fn().mockResolvedValue(undefined),
+        autoScroll: vi.fn().mockResolvedValue(undefined),
+        installInterceptor: vi.fn().mockResolvedValue(undefined),
+        getInterceptedRequests: vi.fn().mockResolvedValue([]),
+        getCookies: vi.fn().mockResolvedValue([]),
+        screenshot: vi.fn().mockResolvedValue(''),
+        waitForCapture: vi.fn().mockResolvedValue(undefined),
+    };
+}
+describe('substack utils wait selectors', () => {
+    it('waits for both feed link shapes before scraping the feed', async () => {
+        const page = createPageMock([]);
+        await loadSubstackFeed(page, 'https://substack.com/', 5);
+        expect(page.wait).toHaveBeenCalledWith({
+            selector: __test__.FEED_POST_LINK_SELECTOR,
+            timeout: 5,
+        });
+    });
+    it('waits for archive post links before scraping archive pages', async () => {
+        const page = createPageMock([]);
+        await loadSubstackArchive(page, 'https://example.substack.com', 5);
+        expect(page.wait).toHaveBeenCalledWith({
+            selector: __test__.ARCHIVE_POST_LINK_SELECTOR,
+            timeout: 5,
+        });
+    });
+});