jackwener--opencli/clis/1688/search.js

import { CommandExecutionError, EmptyResultError } from '@jackwener/opencli/errors';
import { cli, Strategy } from '@jackwener/opencli/registry';
import { FACTORY_BADGE_PATTERNS, SERVICE_BADGE_PATTERNS, assertAuthenticatedState, buildProvenance, buildSearchUrl, canonicalizeItemUrl, canonicalizeSellerUrl, cleanText, extractBadges, extractLocation, extractMemberId, extractOfferId, extractShopId, gotoAndReadState, parseMoqText, parsePriceText, SEARCH_LIMIT_DEFAULT, SEARCH_LIMIT_MAX, parseSearchLimit, uniqueNonEmpty, } from './shared.js';
const SEARCH_ITEM_URL_PATTERNS = [
    'detail.1688.com/offer/',
    'detail.m.1688.com/page/index.html?offerId=',
];
const MAX_SEARCH_PAGES = 12;
function normalizeSearchCandidate(candidate, sourceUrl) {
    const canonicalItemUrl = canonicalizeItemUrl(cleanText(candidate.item_url));
    const containerText = cleanText(candidate.container_text);
    const priceText = firstNonEmpty([
        normalizeInlineText(candidate.price_text),
        normalizeInlineText(extractPriceText(candidate.hover_price_text)),
    ]);
    const priceRange = parsePriceText(priceText || containerText);
    const moq = parseMoqText(firstNonEmpty([
        normalizeInlineText(candidate.moq_text),
        normalizeInlineText(extractMoqText(containerText)),
    ]));
    const canonicalSellerUrl = canonicalizeSellerUrl(cleanText(candidate.seller_url));
    const evidenceText = uniqueNonEmpty([
        containerText,
        ...(candidate.desc_rows ?? []),
        ...(candidate.tag_items ?? []),
        ...(candidate.hover_items ?? []),
    ]).join('\n');
    const badges = extractBadges(evidenceText, [...FACTORY_BADGE_PATTERNS, ...SERVICE_BADGE_PATTERNS]);
    const salesText = firstNonEmpty([
        extractSalesText(candidate.sales_text),
        extractSalesText(containerText),
    ]);
    const returnRateText = extractReturnRateText([...(candidate.tag_items ?? []), ...(candidate.hover_items ?? [])]);
    const provenance = buildProvenance(sourceUrl);
    return {
        rank: 0,
        offer_id: extractOfferId(canonicalItemUrl ?? '') ?? null,
        member_id: extractMemberId(canonicalSellerUrl ?? '') ?? null,
        shop_id: extractShopId(canonicalSellerUrl ?? '') ?? null,
        title: cleanText(candidate.title) || firstWord(containerText) || null,
        item_url: canonicalItemUrl,
        seller_name: cleanText(candidate.seller_name) || null,
        seller_url: canonicalSellerUrl,
        price_text: priceRange.price_text || null,
        price_min: priceRange.price_min,
        price_max: priceRange.price_max,
        currency: priceRange.currency,
        moq_text: moq.moq_text || null,
        moq_value: moq.moq_value,
        location: extractLocation(containerText),
        badges,
        sales_text: salesText || null,
        return_rate_text: returnRateText,
        source_url: provenance.source_url,
        fetched_at: provenance.fetched_at,
        strategy: provenance.strategy,
    };
}
function extractMoqText(text) {
    const normalized = normalizeInlineText(text);
    return normalized.match(/\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)\s*起批/i)?.[0]
        ?? normalized.match(/≥\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)?/i)?.[0]
        ?? normalized.match(/\d+(?:\.\d+)?\s*(?:~|-|至|到)\s*\d+(?:\.\d+)?\s*(件|个|套|箱|包|双|台|把|只)/i)?.[0]
        ?? '';
}
function extractPriceText(text) {
    const normalized = normalizeInlineText(text);
    return normalized.match(/[¥$€]\s*\d+(?:\.\d+)?/)?.[0] ?? '';
}
function extractSalesText(text) {
    const normalized = normalizeInlineText(text);
    if (!normalized)
        return '';
    if (/^\d+(?:\.\d+)?\+?\s*(件|套|个|单)$/.test(normalized)) {
        return normalized;
    }
    const match = normalized.match(/(?:已售|销量|售)\s*\d+(?:\.\d+)?\+?\s*(件|套|个|单)?/);
    return match ? cleanText(match[0]) : '';
}
function firstWord(text) {
    return text.split(/\s+/).find(Boolean) ?? '';
}
function firstNonEmpty(values) {
    return values.map((value) => cleanText(value)).find(Boolean) ?? '';
}
function normalizeInlineText(text) {
    return cleanText(text)
        .replace(/([¥$€])\s+(?=\d)/g, '$1')
        .replace(/(\d)\s*\.\s*(\d)/g, '$1.$2')
        .replace(/\s*([~-])\s*/g, '$1')
        .trim();
}
function extractReturnRateText(values) {
    return uniqueNonEmpty(values.map((value) => normalizeInlineText(value)))
        .find((value) => /^回头率\s*\d+(?:\.\d+)?%$/.test(value))
        ?? null;
}
function buildDedupeKey(row) {
    if (row.offer_id)
        return `offer:${row.offer_id}`;
    if (row.item_url)
        return `url:${row.item_url}`;
    return null;
}
async function readSearchPayload(page, url) {
    const state = await gotoAndReadState(page, url, 2500, 'search');
    assertAuthenticatedState(state, 'search');
    const payload = await page.evaluate(`
    (() => {
      const normalizeText = (value) => (value || '').replace(/\\s+/g, ' ').trim();
      const normalizeUrl = (href) => {
        if (!href) return '';
        try {
          return new URL(href, window.location.href).toString();
        } catch {
          return '';
        }
      };
      const isItemHref = (href) => ${JSON.stringify(SEARCH_ITEM_URL_PATTERNS)}
        .some((pattern) => (href || '').includes(pattern));
      const uniqueTexts = (values) => [...new Set(values.map((value) => normalizeText(value)).filter(Boolean))];
      const collectTexts = (root, selector) => uniqueTexts(
        Array.from(root.querySelectorAll(selector)).map((node) => node.innerText || node.textContent || ''),
      );
      const firstText = (root, selectors) => {
        for (const selector of selectors) {
          const node = root.querySelector(selector);
          const value = normalizeText(node ? node.innerText || node.textContent || '' : '');
          if (value) return value;
        }
        return '';
      };
      const findMoqText = (values, priceText) => {
        const moqPattern = /(≥\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)?)|(\\d+(?:\\.\\d+)?\\s*(?:~|-|至|到)\\s*\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只))|(\\d+(?:\\.\\d+)?\\s*(件|个|套|箱|包|双|台|把|只)\\s*起批)/i;
        return values.find((value) => moqPattern.test(value))
          || normalizeText(priceText).match(moqPattern)?.[0]
          || '';
      };
      const isSellerHref = (href) => {
        if (!href) return false;
        try {
          const url = new URL(href, window.location.href);
          const host = url.hostname || '';
          if (!host.endsWith('.1688.com')) return false;
          if (
            host === 's.1688.com'
            || host === 'r.1688.com'
            || host === 'air.1688.com'
            || host === 'detail.1688.com'
            || host === 'detail.m.1688.com'
            || host === 'dj.1688.com'
          ) {
            return false;
          }
          return true;
        } catch {
          return false;
        }
      };
      const pickContainer = (anchor) => {
        let node = anchor;
        while (node && node !== document.body) {
          const text = normalizeText(node.innerText || node.textContent || '');
          if (text.length >= 40 && text.length <= 2000) {
            return node;
          }
          node = node.parentElement;
        }
        return anchor;
      };
      const collectCandidates = () => {
        const anchors = Array.from(document.querySelectorAll('a')).filter((anchor) => isItemHref(anchor.href || ''));
        const seen = new Set();
        const items = [];
        for (const anchor of anchors) {
          const href = anchor.href || '';
          if (!href || seen.has(href)) continue;
          seen.add(href);

          const container = pickContainer(anchor);
          const tagItems = collectTexts(container, '.offer-tag-row .offer-desc-item');
          const hoverItems = collectTexts(container, '.offer-hover-wrapper .offer-desc-item');
          const sellerAnchor = Array.from(container.querySelectorAll('a'))
            .find((link) => isSellerHref(link.href || ''));
          const hoverPriceText = firstText(container, [
            '.offer-hover-wrapper .hover-price-item',
            '.offer-hover-wrapper .price-item',
          ]);

          items.push({
            item_url: href,
            title: firstText(container, ['.offer-title-row .title-text', '.offer-title-row'])
              || normalizeText(anchor.innerText || anchor.textContent || ''),
            container_text: normalizeText(container.innerText || container.textContent || ''),
            desc_rows: collectTexts(container, '.offer-desc-row'),
            price_text: firstText(container, ['.offer-price-row .price-item']),
            sales_text: firstText(container, ['.offer-price-row .col-desc_after', '.offer-desc-row .col-desc_after']),
            hover_price_text: hoverPriceText,
            moq_text: findMoqText(hoverItems, hoverPriceText),
            tag_items: tagItems,
            hover_items: hoverItems,
            seller_name: sellerAnchor ? normalizeText(sellerAnchor.innerText || sellerAnchor.textContent || '') : null,
            seller_url: sellerAnchor ? sellerAnchor.href : null,
          });
        }
        return items;
      };
      const findNextUrl = () => {
        const selectors = [
          'a.fui-next:not(.disabled)',
          'a.next-pagination-item:not(.disabled)',
          'a[rel="next"]:not(.disabled)',
          'a[data-role="next"]:not(.disabled)',
        ];
        for (const selector of selectors) {
          const node = document.querySelector(selector);
          if (!node) continue;
          const href = normalizeUrl(node.getAttribute('href') || node.href || '');
          if (href) return href;
        }
        const textBased = Array.from(document.querySelectorAll('a'))
          .find((node) => /下一页|next/i.test(normalizeText(node.textContent || '')));
        if (!textBased) return '';
        return normalizeUrl(textBased.getAttribute('href') || textBased.href || '');
      };

      return {
        href: window.location.href,
        title: document.title || '',
        bodyText: document.body ? document.body.innerText || '' : '',
        next_url: findNextUrl(),
        candidates: collectCandidates(),
      };
    })()
  `);
    if (!payload || typeof payload !== 'object') {
        throw new CommandExecutionError('1688 search page did not return a readable payload', 'Open the same query in Chrome and verify the page is fully loaded before retrying.');
    }
    return payload;
}
async function collectSearchRows(page, query, limit) {
    const rowsByKey = new Map();
    const seenPages = new Set();
    let nextUrl = buildSearchUrl(query);
    let pageCount = 0;
    while (nextUrl && rowsByKey.size < limit && pageCount < MAX_SEARCH_PAGES) {
        if (seenPages.has(nextUrl))
            break;
        seenPages.add(nextUrl);
        pageCount += 1;
        const payload = await readSearchPayload(page, nextUrl);
        const sourceUrl = cleanText(payload.href) || nextUrl;
        const candidates = Array.isArray(payload.candidates) ? payload.candidates : [];
        for (const candidate of candidates) {
            const row = normalizeSearchCandidate(candidate, sourceUrl);
            const dedupeKey = buildDedupeKey(row);
            if (!dedupeKey || rowsByKey.has(dedupeKey))
                continue;
            rowsByKey.set(dedupeKey, row);
            if (rowsByKey.size >= limit)
                break;
        }
        const candidateNextUrl = cleanText(payload.next_url);
        if (!candidateNextUrl || candidateNextUrl === sourceUrl)
            break;
        nextUrl = candidateNextUrl;
    }
    if (rowsByKey.size === 0) {
        throw new EmptyResultError('1688 search', 'No visible results were extracted. Retry with a different query or open the same search page in Chrome first.');
    }
    return [...rowsByKey.values()]
        .slice(0, limit)
        .map((row, index) => ({ ...row, rank: index + 1 }));
}
cli({
    site: '1688',
    name: 'search',
    access: 'read',
    description: '1688 商品搜索（结果候选、卖家链接、价格/MOQ/销量文本）',
    domain: 'www.1688.com',
    strategy: Strategy.COOKIE,
    navigateBefore: false,
    args: [
        {
            name: 'query',
            required: true,
            positional: true,
            help: '搜索关键词，如 "置物架"',
        },
        {
            name: 'limit',
            type: 'int',
            default: SEARCH_LIMIT_DEFAULT,
            help: `结果数量上限（默认 ${SEARCH_LIMIT_DEFAULT}，最大 ${SEARCH_LIMIT_MAX}）`,
        },
    ],
    columns: ['rank', 'offer_id', 'title', 'item_url', 'price_text', 'moq_text', 'seller_name', 'member_id', 'location'],
    func: async (page, kwargs) => {
        const query = String(kwargs.query ?? '');
        const limit = parseSearchLimit(kwargs.limit);
        return collectSearchRows(page, query, limit);
    },
});
export const __test__ = {
    normalizeSearchCandidate,
    extractMoqText,
    extractSalesText,
    firstWord,
    buildDedupeKey,
};