wehub-resource-sync
9b395f5cc3
E2E Headed Chrome / e2e-headed (macos-15) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (ubuntu-latest) (push) Has been cancelled
E2E Headed Chrome / e2e-headed (windows-latest) (push) Has been cancelled
CI / build (macos-latest) (push) Has been cancelled
CI / build (ubuntu-latest) (push) Has been cancelled
CI / build (windows-latest) (push) Has been cancelled
CI / unit-test (push) Has been cancelled
CI / bun-test (push) Has been cancelled
CI / adapter-test (push) Has been cancelled
CI / smoke-test (macos-latest) (push) Has been cancelled
CI / smoke-test (ubuntu-latest) (push) Has been cancelled
Security Audit / audit (push) Has been cancelled
Build Chrome Extension / build (push) Has been cancelled
Trigger Website Rebuild (Docs Updated) / dispatch (push) Has been cancelled
410 lines
16 KiB
JavaScript
410 lines
16 KiB
JavaScript
import { cli, Strategy } from '@jackwener/opencli/registry';
|
|
import { ArgumentError, AuthRequiredError, EmptyResultError, TimeoutError } from '@jackwener/opencli/errors';
|
|
import fs from 'node:fs';
|
|
import {
|
|
normalizeConversationRows,
|
|
normalizeManifestRows,
|
|
requireBooleanEvaluateResult,
|
|
requireObjectEvaluateResult,
|
|
} from './export-utils.js';
|
|
|
|
const GROK_DOMAIN = 'grok.com';
|
|
const GROK_URL = 'https://grok.com/';
|
|
|
|
function normalizeInteger(value, defaultValue, label, { min = 0, max = Number.MAX_SAFE_INTEGER } = {}) {
|
|
const raw = value ?? defaultValue;
|
|
const n = Number(raw);
|
|
if (!Number.isInteger(n)) {
|
|
throw new ArgumentError(label, `must be an integer`);
|
|
}
|
|
if (n < min) {
|
|
throw new ArgumentError(label, `must be >= ${min}`);
|
|
}
|
|
if (n > max) {
|
|
throw new ArgumentError(label, `must be <= ${max}`);
|
|
}
|
|
return n;
|
|
}
|
|
|
|
async function waitRandom(page, minMs, maxMs) {
|
|
if (maxMs <= 0) return;
|
|
const span = Math.max(0, maxMs - minMs);
|
|
const ms = minMs + Math.floor(Math.random() * (span + 1));
|
|
if (ms > 0) await page.wait(ms / 1000);
|
|
}
|
|
|
|
function readManifest(manifestPath, { offset, limit }) {
|
|
const path = String(manifestPath || '').trim();
|
|
if (!path) return null;
|
|
let parsed;
|
|
try {
|
|
parsed = JSON.parse(fs.readFileSync(path, 'utf8'));
|
|
} catch (error) {
|
|
throw new ArgumentError('manifestPath', `failed to read JSON manifest: ${error?.message || error}`);
|
|
}
|
|
const rows = normalizeManifestRows(parsed);
|
|
const sliced = limit ? rows.slice(offset, offset + limit) : rows.slice(offset);
|
|
if (!sliced.length) {
|
|
throw new EmptyResultError('grok export-all', `No manifest rows after offset=${offset}, limit=${limit}`);
|
|
}
|
|
return sliced;
|
|
}
|
|
|
|
async function collectHistory(page, { offset, limit, maxScrolls }) {
|
|
await page.goto(GROK_URL);
|
|
await page.wait(2);
|
|
const rawResult = await page.evaluate(`(async () => {
|
|
const targetLimit = ${JSON.stringify(limit > 0 ? offset + limit : 0)};
|
|
const maxScrolls = ${JSON.stringify(maxScrolls)};
|
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
const isVisible = (node) => {
|
|
if (!(node instanceof Element)) return false;
|
|
const style = window.getComputedStyle(node);
|
|
if (style.visibility === 'hidden' || style.display === 'none') return false;
|
|
const rect = node.getBoundingClientRect();
|
|
return rect.width > 0 && rect.height > 0;
|
|
};
|
|
const hasHistoryEntry = Boolean(document.querySelector('a[href^="/c/"]'));
|
|
const hasHistoryLauncher = Array.from(document.querySelectorAll('button, [role="button"]'))
|
|
.some((node) => isVisible(node) && /^(查看全部|show all|view all)$/i.test((node.textContent || '').trim()));
|
|
const signInCta = Array.from(document.querySelectorAll('button, a'))
|
|
.some((node) => isVisible(node) && /^(sign in|log in)$/i.test((node.textContent || '').trim()));
|
|
if (signInCta && !hasHistoryEntry && !hasHistoryLauncher) {
|
|
return { ok: false, code: 'AUTH' };
|
|
}
|
|
|
|
const clickAllHistory = () => {
|
|
const buttons = Array.from(document.querySelectorAll('button, [role="button"]'))
|
|
.filter((node) => node instanceof HTMLElement && isVisible(node));
|
|
const target = buttons.find((node) => /^(查看全部|show all|view all)$/i.test((node.textContent || '').trim()));
|
|
if (!target) return false;
|
|
target.click();
|
|
return true;
|
|
};
|
|
|
|
if (!document.querySelector('[role="listbox"] a[href^="/c/"]')) {
|
|
clickAllHistory();
|
|
}
|
|
|
|
let listbox = null;
|
|
for (let attempt = 0; attempt < 30; attempt += 1) {
|
|
listbox = document.querySelector('[role="listbox"]');
|
|
if (listbox && listbox.querySelector('a[href^="/c/"]')) break;
|
|
await sleep(250);
|
|
}
|
|
if (!listbox || !listbox.querySelector('a[href^="/c/"]')) {
|
|
return { ok: false, code: 'NO_DIALOG' };
|
|
}
|
|
|
|
const scroller = Array.from(listbox.querySelectorAll('*'))
|
|
.find((node) => node.scrollHeight > node.clientHeight + 20) || listbox;
|
|
const seen = new Map();
|
|
const collect = () => {
|
|
for (const a of Array.from(listbox.querySelectorAll('a[href^="/c/"]'))) {
|
|
const href = a.getAttribute('href') || '';
|
|
const match = href.match(/^\\/c\\/([0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12})$/i);
|
|
if (!match) continue;
|
|
const id = match[1].toLowerCase();
|
|
const option = a.closest('[role="option"]') || a.parentElement;
|
|
const lines = (option?.innerText || option?.textContent || a.textContent || '')
|
|
.split(/\\n+/)
|
|
.map((line) => line.trim())
|
|
.filter(Boolean);
|
|
seen.set(id, {
|
|
id,
|
|
title: lines[0] || '',
|
|
date: lines.slice(1).find(Boolean) || '',
|
|
url: 'https://grok.com/c/' + id,
|
|
});
|
|
}
|
|
};
|
|
|
|
scroller.scrollTop = 0;
|
|
scroller.dispatchEvent(new Event('scroll', { bubbles: true }));
|
|
await sleep(250);
|
|
|
|
let lastCount = -1;
|
|
let lastHeight = -1;
|
|
let stableRounds = 0;
|
|
for (let attempt = 0; attempt < maxScrolls; attempt += 1) {
|
|
collect();
|
|
if (targetLimit > 0 && seen.size >= targetLimit) break;
|
|
scroller.scrollTop = scroller.scrollHeight;
|
|
scroller.dispatchEvent(new Event('scroll', { bubbles: true }));
|
|
await sleep(350);
|
|
collect();
|
|
if (targetLimit > 0 && seen.size >= targetLimit) break;
|
|
if (seen.size === lastCount && scroller.scrollHeight === lastHeight) {
|
|
stableRounds += 1;
|
|
} else {
|
|
stableRounds = 0;
|
|
lastCount = seen.size;
|
|
lastHeight = scroller.scrollHeight;
|
|
}
|
|
if (targetLimit === 0 && stableRounds >= 8) break;
|
|
}
|
|
|
|
return { ok: true, rows: Array.from(seen.values()) };
|
|
})()`);
|
|
|
|
const result = requireObjectEvaluateResult(rawResult, 'grok export-all history dialog');
|
|
if (result.ok !== true) {
|
|
if (result?.code === 'AUTH') {
|
|
throw new AuthRequiredError(GROK_DOMAIN, 'Sign in to grok.com in the browser, then retry.');
|
|
}
|
|
if (result?.code === 'NO_DIALOG') {
|
|
throw new TimeoutError('grok export-all history dialog', 8);
|
|
}
|
|
throw new EmptyResultError('grok export-all', 'No Grok conversation history was visible.');
|
|
}
|
|
const validRows = normalizeConversationRows(result.rows, 'grok export-all history dialog');
|
|
if (!validRows.length) {
|
|
throw new EmptyResultError('grok export-all', 'No Grok conversations found in the signed-in account history.');
|
|
}
|
|
return limit ? validRows.slice(offset, offset + limit) : validRows.slice(offset);
|
|
}
|
|
|
|
async function readConversation(page, conversation, { pageTimeoutMs, pageScrolls, delayMinMs, delayMaxMs }) {
|
|
await page.goto(conversation.url);
|
|
const startedAt = Date.now();
|
|
let loaded = false;
|
|
|
|
while (Date.now() - startedAt < pageTimeoutMs) {
|
|
let loadedPayload;
|
|
try {
|
|
loadedPayload = await page.evaluate(`(() => {
|
|
return Boolean(document.querySelector('[data-testid="user-message"], [data-testid="assistant-message"]'));
|
|
})()`);
|
|
loaded = requireBooleanEvaluateResult(loadedPayload, 'grok export-all page load check');
|
|
} catch (error) {
|
|
return {
|
|
status: 'failed',
|
|
error: `Page load check failed: ${error?.message || error}`,
|
|
messageCount: 0,
|
|
messagesJson: '[]',
|
|
};
|
|
}
|
|
if (loaded) break;
|
|
await page.wait(1);
|
|
}
|
|
if (!loaded) {
|
|
return {
|
|
status: 'empty',
|
|
error: 'No visible message bubbles after waiting for page load.',
|
|
messageCount: 0,
|
|
messagesJson: '[]',
|
|
};
|
|
}
|
|
|
|
await waitRandom(page, delayMinMs, delayMaxMs);
|
|
|
|
let rawResult;
|
|
let result;
|
|
try {
|
|
rawResult = await page.evaluate(`(async () => {
|
|
const maxScrolls = ${JSON.stringify(pageScrolls)};
|
|
const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
|
|
const isVisible = (node) => {
|
|
if (!(node instanceof Element)) return false;
|
|
const style = window.getComputedStyle(node);
|
|
if (style.visibility === 'hidden' || style.display === 'none') return false;
|
|
const rect = node.getBoundingClientRect();
|
|
return rect.width > 0 && rect.height > 0;
|
|
};
|
|
const messageSelector = '[data-testid="user-message"], [data-testid="assistant-message"]';
|
|
const findScrollableAncestors = () => {
|
|
const first = document.querySelector(messageSelector);
|
|
const out = [document.scrollingElement || document.documentElement];
|
|
let node = first ? first.parentElement : null;
|
|
while (node && node !== document.body) {
|
|
if (node.scrollHeight > node.clientHeight + 20) out.push(node);
|
|
node = node.parentElement;
|
|
}
|
|
return Array.from(new Set(out));
|
|
};
|
|
const scrollables = findScrollableAncestors();
|
|
const countMessages = () => Array.from(document.querySelectorAll(messageSelector)).filter(isVisible).length;
|
|
|
|
let lastCount = -1;
|
|
let lastBottom = -1;
|
|
let stableRounds = 0;
|
|
for (let attempt = 0; attempt < maxScrolls && stableRounds < 5; attempt += 1) {
|
|
for (const scroller of scrollables) {
|
|
scroller.scrollTop = scroller.scrollHeight;
|
|
scroller.dispatchEvent(new Event('scroll', { bubbles: true }));
|
|
}
|
|
window.scrollTo(0, document.documentElement.scrollHeight || document.body.scrollHeight);
|
|
await sleep(300);
|
|
const count = countMessages();
|
|
const bottom = Math.max(
|
|
document.documentElement.scrollHeight || 0,
|
|
document.body.scrollHeight || 0,
|
|
...scrollables.map((node) => node.scrollHeight || 0),
|
|
);
|
|
if (count === lastCount && bottom === lastBottom) {
|
|
stableRounds += 1;
|
|
} else {
|
|
stableRounds = 0;
|
|
lastCount = count;
|
|
lastBottom = bottom;
|
|
}
|
|
}
|
|
|
|
const findResponseId = (node) => {
|
|
let parent = node.parentElement;
|
|
while (parent && parent !== document.body) {
|
|
const id = parent.getAttribute('id') || '';
|
|
if (id.startsWith('response-')) return id.slice('response-'.length);
|
|
parent = parent.parentElement;
|
|
}
|
|
return '';
|
|
};
|
|
const messages = [];
|
|
let position = 0;
|
|
for (const node of Array.from(document.querySelectorAll(messageSelector))) {
|
|
if (!(node instanceof HTMLElement) || !isVisible(node)) continue;
|
|
const isAssistant = node.getAttribute('data-testid') === 'assistant-message';
|
|
const text = (node.innerText || node.textContent || '').replace(/\\s+/g, ' ').trim();
|
|
const html = node.innerHTML || '';
|
|
if (!text && !html) continue;
|
|
messages.push({
|
|
messageIndex: messages.length + 1,
|
|
messageId: findResponseId(node) || ('pos-' + position),
|
|
messageRole: isAssistant ? 'assistant' : 'user',
|
|
messageText: text,
|
|
});
|
|
position += 1;
|
|
}
|
|
return { messages };
|
|
})()`);
|
|
result = requireObjectEvaluateResult(rawResult, 'grok export-all conversation reader');
|
|
} catch (error) {
|
|
return {
|
|
status: 'failed',
|
|
error: `Conversation reader failed: ${error?.message || error}`,
|
|
messageCount: 0,
|
|
messagesJson: '[]',
|
|
};
|
|
}
|
|
if (!Array.isArray(result.messages)) {
|
|
return {
|
|
status: 'failed',
|
|
error: 'Conversation reader returned malformed message rows.',
|
|
messageCount: 0,
|
|
messagesJson: '[]',
|
|
};
|
|
}
|
|
const messages = [];
|
|
for (let index = 0; index < result.messages.length; index += 1) {
|
|
const message = result.messages[index];
|
|
if (!message || typeof message !== 'object' || Array.isArray(message)) {
|
|
return {
|
|
status: 'failed',
|
|
error: `Conversation reader returned malformed message row ${index + 1}.`,
|
|
messageCount: 0,
|
|
messagesJson: '[]',
|
|
};
|
|
}
|
|
const messageId = String(message.messageId || '').trim();
|
|
const messageText = String(message.messageText || '').trim();
|
|
if (!messageId || !messageText || (message.messageRole !== 'assistant' && message.messageRole !== 'user')) {
|
|
return {
|
|
status: 'failed',
|
|
error: `Conversation reader returned malformed message row ${index + 1}.`,
|
|
messageCount: 0,
|
|
messagesJson: '[]',
|
|
};
|
|
}
|
|
messages.push({
|
|
messageIndex: Number.isInteger(message.messageIndex) ? message.messageIndex : index + 1,
|
|
messageId,
|
|
messageRole: message.messageRole,
|
|
messageText,
|
|
});
|
|
}
|
|
if (!messages.length) {
|
|
return {
|
|
status: 'empty',
|
|
error: 'Page loaded, but no visible message text was found after scrolling to bottom.',
|
|
messageCount: 0,
|
|
messagesJson: '[]',
|
|
};
|
|
}
|
|
return {
|
|
status: 'ok',
|
|
error: null,
|
|
messageCount: messages.length,
|
|
messagesJson: JSON.stringify(messages),
|
|
};
|
|
}
|
|
|
|
export const grokExportAllCommand = cli({
|
|
site: 'grok',
|
|
name: 'export-all',
|
|
description: 'Export Grok conversation history and each conversation transcript',
|
|
access: 'read',
|
|
example: 'opencli grok export-all --limit 5 -f json',
|
|
domain: GROK_DOMAIN,
|
|
strategy: Strategy.COOKIE,
|
|
browser: true,
|
|
siteSession: 'persistent',
|
|
navigateBefore: false,
|
|
args: [
|
|
{ name: 'limit', type: 'int', default: 0, help: 'Max conversations to export; 0 means all loaded history' },
|
|
{ name: 'offset', type: 'int', default: 0, help: 'Skip this many conversations before exporting' },
|
|
{ name: 'manifestPath', type: 'string', default: '', help: 'Optional grok/export JSON manifest path; skips history dialog and visits listed /c pages directly' },
|
|
{ name: 'maxScrolls', type: 'int', default: 80, help: 'Max history-list scroll rounds when limit is 0 (max 500)' },
|
|
{ name: 'pageScrolls', type: 'int', default: 30, help: 'Max per-conversation scroll-to-bottom rounds (max 200)' },
|
|
{ name: 'pageTimeoutMs', type: 'int', default: 30000, help: 'Max wait for each conversation page to show messages' },
|
|
{ name: 'delayMinMs', type: 'int', default: 0, help: 'Minimum polite delay after a conversation page loads' },
|
|
{ name: 'delayMaxMs', type: 'int', default: 5000, help: 'Maximum polite delay after a conversation page loads' },
|
|
],
|
|
columns: ['index', 'id', 'title', 'date', 'url', 'status', 'messageCount', 'error', 'messagesJson'],
|
|
func: async (page, kwargs) => {
|
|
const limit = normalizeInteger(kwargs.limit, 0, 'limit', { min: 0 });
|
|
const offset = normalizeInteger(kwargs.offset, 0, 'offset', { min: 0 });
|
|
const maxScrolls = normalizeInteger(kwargs.maxScrolls, 80, 'maxScrolls', { min: 1, max: 500 });
|
|
const pageScrolls = normalizeInteger(kwargs.pageScrolls, 30, 'pageScrolls', { min: 1, max: 200 });
|
|
const pageTimeoutMs = normalizeInteger(kwargs.pageTimeoutMs, 30000, 'pageTimeoutMs', { min: 5000, max: 180000 });
|
|
const delayMinMs = normalizeInteger(kwargs.delayMinMs, 0, 'delayMinMs', { min: 0, max: 60000 });
|
|
const delayMaxMs = normalizeInteger(kwargs.delayMaxMs, 5000, 'delayMaxMs', { min: 0, max: 60000 });
|
|
if (delayMaxMs < delayMinMs) {
|
|
throw new ArgumentError('delayMaxMs', 'must be >= delayMinMs');
|
|
}
|
|
|
|
const conversations = readManifest(kwargs.manifestPath, { offset, limit })
|
|
|| await collectHistory(page, { offset, limit, maxScrolls });
|
|
const rows = [];
|
|
for (let i = 0; i < conversations.length; i += 1) {
|
|
const conversation = conversations[i];
|
|
const transcript = await readConversation(page, conversation, {
|
|
pageTimeoutMs,
|
|
pageScrolls,
|
|
delayMinMs,
|
|
delayMaxMs,
|
|
});
|
|
rows.push({
|
|
index: offset + i + 1,
|
|
id: conversation.id,
|
|
title: conversation.title || null,
|
|
date: conversation.date || null,
|
|
url: conversation.url,
|
|
status: transcript.status,
|
|
messageCount: transcript.messageCount,
|
|
error: transcript.error,
|
|
messagesJson: transcript.messagesJson,
|
|
});
|
|
}
|
|
|
|
if (!rows.length) {
|
|
throw new EmptyResultError('grok export-all', 'No Grok conversations were exported.');
|
|
}
|
|
return rows;
|
|
},
|
|
});
|
|
|
|
export const __test__ = {
|
|
normalizeInteger,
|
|
readManifest,
|
|
};
|