jackwener--opencli/clis/tiktok/explore.js

// Browse TikTok "For You" / explore feed via page-context API.
//
// Replaces legacy DOM-link scraping (`querySelectorAll('a[href*="/video/"]')`).
// We borrow the live page session: read `__UNIVERSAL_DATA_FOR_REHYDRATION__` for
// the warm `webapp.recommend-feed` snapshot, then extend with `/api/recommend/item_list/`
// when a larger `--limit` is requested. Stays inside the browser, so the session
// cookies + msToken go along automatically.

import { cli, Strategy } from '@jackwener/opencli/registry';
import {
    EmptyResultError,
} from '@jackwener/opencli/errors';
import {
    BROWSER_HELPERS,
    MAX_PAGES,
    SERVER_PAGE_MAX,
    TIKTOK_AID,
    requireLimit,
    throwTikTokPageContextError,
    VIDEO_ITEM_NORMALIZER,
} from './utils.js';

const DEFAULT_LIMIT = 20;
const MAX_LIMIT = 120;

function buildExploreScript(limit) {
    return `
(async () => {
  const limit = ${Number(limit)};
  const maxPages = ${MAX_PAGES};
  const SERVER_PAGE_MAX = ${SERVER_PAGE_MAX};
  // limit was validated upfront (requireLimit). pageSize is the per-request
  // cap TikTok accepts; we still loop until \`dedup.size < limit\` is satisfied.
  const pageSize = limit < SERVER_PAGE_MAX ? limit : SERVER_PAGE_MAX;
  const aid = ${JSON.stringify(TIKTOK_AID)};

  ${BROWSER_HELPERS}
  ${VIDEO_ITEM_NORMALIZER}

  function collectFromState(root) {
    if (!root) return [];
    const out = [];
    walkObjects(root, (node) => {
      if (Array.isArray(node)) return false;
      if (Array.isArray(node.itemList)) {
        for (const item of node.itemList) out.push(item);
      }
      if (Array.isArray(node.items)) {
        for (const item of node.items) out.push(item);
      }
      return false;
    });
    return out;
  }

  const dedup = new Map();
  const universal = findUniversalData();
  for (const item of collectFromState(universal)) {
    const row = normalizeVideoItem(item, dedup.size + 1);
    if (row && !dedup.has(row.id)) dedup.set(row.id, row);
  }

  const msToken = getCookie('msToken');
  let apiFailure = null;
  if (dedup.size < limit) {
    let cursor = 0;
    for (let page = 0; page < maxPages && dedup.size < limit; page += 1) {
      const params = new URLSearchParams({
        aid,
        count: String(pageSize),
        from_page: 'fyp',
        cursor: String(cursor),
      });
      if (msToken) params.set('msToken', msToken);
      try {
        const data = await fetchJson('/api/recommend/item_list/?' + params.toString());
        assertTikTokApiSuccess(data, 'recommend');
        const items = Array.isArray(data.itemList) ? data.itemList : [];
        for (const item of items) {
          const row = normalizeVideoItem(item, dedup.size + 1);
          if (row && !dedup.has(row.id)) dedup.set(row.id, row);
        }
        if (data.hasMore !== true && items.length === 0) break;
        cursor = asNumber(data.cursor) ?? cursor + items.length;
      } catch (error) {
        apiFailure = error instanceof Error ? error.message : String(error);
        break;
      }
    }
  }

  const rows = Array.from(dedup.values())
    .slice(0, limit)
    .map((row, index) => ({ ...row, index: index + 1 }));

  if (rows.length === 0) {
    const suffix = apiFailure ? ' (recommend API failed: ' + apiFailure + ')' : '';
    throw new Error('No videos found on /explore' + suffix);
  }
  return rows;
})()
`;
}

async function listExploreVideos(page, args) {
    const limit = requireLimit(args.limit, { fallback: DEFAULT_LIMIT, max: MAX_LIMIT });
    await page.goto('https://www.tiktok.com/explore', { waitUntil: 'load', settleMs: 5000 });
    let rows;
    try {
        rows = await page.evaluate(buildExploreScript(limit));
    } catch (error) {
        throwTikTokPageContextError(error, {
            authMessage: 'TikTok requires browser access to load the explore feed',
            emptyPattern: /No videos found/,
            emptyTarget: 'tiktok explore',
            failureMessage: 'Failed to load TikTok explore feed',
        });
    }
    if (!Array.isArray(rows) || rows.length === 0) {
        throw new EmptyResultError('tiktok explore', 'TikTok returned an empty recommend feed');
    }
    return rows;
}

export const exploreCommand = cli({
    site: 'tiktok',
    name: 'explore',
    access: 'read',
    description: 'Get trending TikTok videos from the recommend feed via page-context APIs',
    domain: 'www.tiktok.com',
    strategy: Strategy.COOKIE,
    browser: true,
    args: [
        { name: 'limit', type: 'int', default: DEFAULT_LIMIT, help: `Number of videos to return (max ${MAX_LIMIT})` },
    ],
    columns: ['index', 'id', 'author', 'url', 'cover', 'title', 'desc', 'plays', 'likes', 'comments', 'shares', 'createTime'],
    func: listExploreVideos,
});

export const __test__ = {
    buildExploreScript,
    DEFAULT_LIMIT,
    MAX_LIMIT,
};