chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,68 @@
|
||||
# OpenAPI Scripts
|
||||
|
||||
## fetch-with-retry.ts
|
||||
|
||||
Shared `fetchWithRetry` helper used by the data-generation scripts
|
||||
(`generate-toolkits.ts`, `generate-meta-tools.ts`). It wraps the global `fetch`
|
||||
with rate-limit-aware retry/backoff: it retries `429` and transient 5xx
|
||||
responses, honors the `Retry-After` header when present, otherwise falls back to
|
||||
exponential backoff with jitter, and caps attempts so CI still fails fast when
|
||||
the backend is genuinely down.
|
||||
|
||||
This matters because `generate-toolkits.ts` issues ~6500 requests per run
|
||||
(a few catalog pages + 3 per toolkit across a ~2.1k catalog), which exceeds the
|
||||
staging limit of 2000 requests/minute.
|
||||
Before this helper, runs failed with `429`, and `generate-meta-tools.ts` — which
|
||||
runs immediately after — inherited the exhausted rate-limit window.
|
||||
|
||||
## fetch-openapi.mjs
|
||||
|
||||
Fetches the Composio OpenAPI spec and filters it for use in Fumadocs API reference documentation.
|
||||
|
||||
### Usage
|
||||
|
||||
```bash
|
||||
bun run scripts/fetch-openapi.mjs
|
||||
```
|
||||
|
||||
This outputs `public/openapi.json` which is used by `lib/openapi.ts`.
|
||||
|
||||
### Why Filtering is Needed
|
||||
|
||||
The raw OpenAPI spec from `https://backend.composio.dev/api/v3/openapi.json` has issues that break documentation generators:
|
||||
|
||||
1. **Endpoints with multiple tags** - Causes duplicate entries in sidebar
|
||||
2. **Internal endpoints exposed** - CLI, Admin, Profiling endpoints shouldn't be in public docs
|
||||
|
||||
See `OPENAPI_IMPROVEMENTS.md` in the fumadocs root for planned fixes to the spec itself.
|
||||
|
||||
### What Gets Filtered
|
||||
|
||||
#### Ignored Paths
|
||||
These endpoints are completely removed:
|
||||
- `/api/v3/mcp/validate/{uuid}`
|
||||
- `/api/v3/cli/get-session`
|
||||
- `/api/v3/cli/create-session`
|
||||
- `/api/v3/auth/session/logout`
|
||||
|
||||
#### Ignored Tags
|
||||
Endpoints with only these tags are removed:
|
||||
- `CLI`
|
||||
- `Admin`
|
||||
- `Profiling`
|
||||
|
||||
#### Duplicate Prevention
|
||||
If an endpoint has multiple tags, only the first tag is kept. This prevents the same endpoint appearing in multiple sidebar sections.
|
||||
|
||||
### Configuration
|
||||
|
||||
The script uses environment variables:
|
||||
|
||||
| Variable | Default | Description |
|
||||
|----------|---------|-------------|
|
||||
| `OPENAPI_SPEC_URL` | `https://backend.composio.dev/api/v3/openapi.json` | Source OpenAPI spec URL |
|
||||
|
||||
For staging deployments, set:
|
||||
```bash
|
||||
OPENAPI_SPEC_URL=https://staging.composio.dev/api/v3/openapi.json
|
||||
```
|
||||
@@ -0,0 +1,381 @@
|
||||
/**
|
||||
* Build the docs snapshot the Ask AI agent imports (`agent/lib/docs-index.ts`).
|
||||
*
|
||||
* The deployed eve service bundles the `agent/` directory but NOT `content/` or
|
||||
* `public/data/`, so the agent can't read the docs from disk at runtime. This
|
||||
* script snapshots every content page (cleaned Markdown + metadata) and the
|
||||
* toolkit catalog into a generated module the agent imports, so eve bundles it
|
||||
* into the deployed service. It also precomputes BM25-style lexical term stats
|
||||
* for the docs assistant search tool, avoiding cold-start corpus construction in
|
||||
* the deployed function.
|
||||
*
|
||||
* In dev the agent still reads `content/` live when the generated search snapshot
|
||||
* does not match; the bundle is always used when the content tree isn't on disk.
|
||||
*
|
||||
* Self-contained on purpose: it does NOT import `agent/lib/docs.ts` (which
|
||||
* imports the generated file), so it can run from a clean checkout where
|
||||
* `docs-index.ts` doesn't exist yet. The generated file is gitignored and
|
||||
* regenerated before `dev`, `build`, and `types:check`.
|
||||
*
|
||||
* Run manually with `bun scripts/build-agent-index.ts`.
|
||||
*/
|
||||
import { existsSync, readdirSync, readFileSync, writeFileSync } from 'node:fs';
|
||||
import { join, relative } from 'node:path';
|
||||
|
||||
const ROOT = process.cwd();
|
||||
const CONTENT = join(ROOT, 'content');
|
||||
const KNOWLEDGE_FILE = join(ROOT, 'agent', 'knowledge.md');
|
||||
const COLLECTIONS = ['docs', 'reference', 'examples', 'toolkits'] as const;
|
||||
const LEGACY_URL_PATTERNS = [
|
||||
'/docs/tools-direct',
|
||||
'/docs/auth-configuration',
|
||||
'/docs/sessions-vs-direct-execution',
|
||||
];
|
||||
const STOPWORDS = new Set([
|
||||
'the',
|
||||
'a',
|
||||
'an',
|
||||
'and',
|
||||
'or',
|
||||
'but',
|
||||
'is',
|
||||
'are',
|
||||
'was',
|
||||
'were',
|
||||
'be',
|
||||
'been',
|
||||
'to',
|
||||
'of',
|
||||
'in',
|
||||
'on',
|
||||
'for',
|
||||
'with',
|
||||
'as',
|
||||
'at',
|
||||
'by',
|
||||
'from',
|
||||
'how',
|
||||
'do',
|
||||
'does',
|
||||
'did',
|
||||
'what',
|
||||
'why',
|
||||
'when',
|
||||
'which',
|
||||
'who',
|
||||
'can',
|
||||
'i',
|
||||
'you',
|
||||
'it',
|
||||
'this',
|
||||
'that',
|
||||
'these',
|
||||
'those',
|
||||
'my',
|
||||
'your',
|
||||
'we',
|
||||
'they',
|
||||
'use',
|
||||
'using',
|
||||
'work',
|
||||
'works',
|
||||
'about',
|
||||
'into',
|
||||
'so',
|
||||
'if',
|
||||
'me',
|
||||
'get',
|
||||
'set',
|
||||
'up',
|
||||
]);
|
||||
|
||||
// The helpers below mirror agent/lib/docs.ts so generated URLs, cleaned
|
||||
// Markdown, page keys, and BM25 token stats match what the runtime path uses.
|
||||
function isLegacyUrl(url: string): boolean {
|
||||
return LEGACY_URL_PATTERNS.some(p => url === p || url.startsWith(`${p}/`));
|
||||
}
|
||||
|
||||
function urlFromContentPath(absPath: string): string | undefined {
|
||||
const rel = relative(CONTENT, absPath).replace(/\\/g, '/');
|
||||
const withoutExt = rel.replace(/\.mdx?$/, '');
|
||||
const parts = withoutExt.split('/');
|
||||
const collection = parts.shift();
|
||||
if (!collection) return undefined;
|
||||
if (collection === 'docs') return `/docs/${parts.join('/')}`.replace(/\/index$/, '');
|
||||
if (collection === 'examples') return `/examples/${parts.join('/')}`.replace(/\/index$/, '');
|
||||
if (collection === 'reference') return `/reference/${parts.join('/')}`.replace(/\/index$/, '');
|
||||
if (collection === 'toolkits') {
|
||||
if (parts[0] === 'faq') return undefined;
|
||||
return `/toolkits/${parts.join('/')}`.replace(/\/index$/, '');
|
||||
}
|
||||
return undefined;
|
||||
}
|
||||
|
||||
function tokenize(query: string): string[] {
|
||||
return query
|
||||
.toLowerCase()
|
||||
.split(/[^a-z0-9_]+/)
|
||||
.filter(t => t.length > 1 && !STOPWORDS.has(t));
|
||||
}
|
||||
|
||||
function parseFrontmatter(raw: string): {
|
||||
title: string;
|
||||
description: string;
|
||||
legacy: boolean;
|
||||
body: string;
|
||||
} {
|
||||
const match = raw.match(/^---\n([\s\S]*?)\n---\n?([\s\S]*)$/);
|
||||
if (!match) return { title: '', description: '', legacy: false, body: raw };
|
||||
const [, fm, body] = match;
|
||||
const get = (key: string) => {
|
||||
const m = fm.match(new RegExp(`^${key}:\\s*(.+)$`, 'm'));
|
||||
return m ? m[1].trim().replace(/^["']|["']$/g, '') : '';
|
||||
};
|
||||
return {
|
||||
title: get('title'),
|
||||
description: get('description'),
|
||||
legacy: get('legacy') === 'true',
|
||||
body,
|
||||
};
|
||||
}
|
||||
|
||||
function toPlainText(body: string): string {
|
||||
return body
|
||||
.replace(/```[\s\S]*?```/g, ' ')
|
||||
.replace(/<\/?[A-Za-z][^>]*>/g, ' ')
|
||||
.replace(/!\[[^\]]*\]\([^)]*\)/g, ' ')
|
||||
.replace(/\[([^\]]*)\]\([^)]*\)/g, '$1')
|
||||
.replace(/[#>*_`|]/g, ' ')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim();
|
||||
}
|
||||
|
||||
function toCleanMarkdown(raw: string): string {
|
||||
const { body } = parseFrontmatter(raw);
|
||||
return body
|
||||
.replace(/<\/?[A-Za-z][A-Za-z0-9.]*(\s[^>]*)?\/?>/g, '')
|
||||
.replace(/\n{3,}/g, '\n\n')
|
||||
.trim();
|
||||
}
|
||||
|
||||
interface BundlePage {
|
||||
collection: string;
|
||||
url: string;
|
||||
title: string;
|
||||
description: string;
|
||||
legacy: boolean;
|
||||
markdown: string;
|
||||
}
|
||||
|
||||
interface BundleToolkit {
|
||||
slug: string;
|
||||
name?: string;
|
||||
description?: string;
|
||||
category?: string;
|
||||
authSchemes?: string[];
|
||||
}
|
||||
|
||||
interface SearchPage {
|
||||
collection: string;
|
||||
title: string;
|
||||
description: string;
|
||||
url: string;
|
||||
headings: string[];
|
||||
lowerText: string;
|
||||
}
|
||||
|
||||
function pageSearchKey(page: Pick<SearchPage, 'collection' | 'title' | 'url'>): string {
|
||||
return `${page.collection}\u0000${page.url}\u0000${page.title}`;
|
||||
}
|
||||
|
||||
function makeSearchPage(args: {
|
||||
collection: string;
|
||||
title: string;
|
||||
description: string;
|
||||
url: string;
|
||||
body: string;
|
||||
}): SearchPage {
|
||||
const headings = (args.body.match(/^#{1,4}\s+(.+)$/gm) ?? []).map(h =>
|
||||
h
|
||||
.replace(/^#{1,4}\s+/, '')
|
||||
.toLowerCase()
|
||||
.trim()
|
||||
);
|
||||
const text = toPlainText(args.body);
|
||||
|
||||
return {
|
||||
collection: args.collection,
|
||||
title: args.title || args.url,
|
||||
description: args.description,
|
||||
url: args.url,
|
||||
headings,
|
||||
lowerText: `${args.title} ${args.description} ${text}`.toLowerCase(),
|
||||
};
|
||||
}
|
||||
|
||||
function loadKnowledgePages(): BundlePage[] {
|
||||
if (!existsSync(KNOWLEDGE_FILE)) return [];
|
||||
|
||||
const raw = readFileSync(KNOWLEDGE_FILE, 'utf8');
|
||||
const pages: BundlePage[] = [];
|
||||
const sectionRe = /^##\s+(.+?)\s*(?:\(([^)]+)\))?\s*$/gm;
|
||||
const matches = [...raw.matchAll(sectionRe)];
|
||||
|
||||
for (let index = 0; index < matches.length; index++) {
|
||||
const match = matches[index];
|
||||
const title = match[1].trim();
|
||||
const url = (match[2] ?? '').trim() || '/docs';
|
||||
const bodyStart = match.index! + match[0].length;
|
||||
const bodyEnd = index + 1 < matches.length ? matches[index + 1].index! : raw.length;
|
||||
const body = raw.slice(bodyStart, bodyEnd).trim();
|
||||
|
||||
pages.push({
|
||||
collection: 'knowledge',
|
||||
url,
|
||||
title,
|
||||
description: '',
|
||||
legacy: false,
|
||||
markdown: body,
|
||||
});
|
||||
}
|
||||
|
||||
return pages;
|
||||
}
|
||||
|
||||
function searchPageForToolkit(tk: BundleToolkit): SearchPage {
|
||||
const name = tk.name ?? tk.slug;
|
||||
const body = `${tk.description ?? ''} Category: ${tk.category ?? ''}. Toolkit slug: ${tk.slug}.`;
|
||||
|
||||
return makeSearchPage({
|
||||
collection: 'toolkits',
|
||||
title: `${name} toolkit`,
|
||||
description: tk.description ?? '',
|
||||
url: `/toolkits/${tk.slug}`,
|
||||
body,
|
||||
});
|
||||
}
|
||||
|
||||
function termCountsFor(page: SearchPage): Map<string, number> {
|
||||
const counts = new Map<string, number>();
|
||||
|
||||
for (const token of tokenize(page.lowerText)) {
|
||||
counts.set(token, (counts.get(token) ?? 0) + 1);
|
||||
}
|
||||
|
||||
return counts;
|
||||
}
|
||||
|
||||
function buildSearchIndex(searchPages: SearchPage[]) {
|
||||
const entries = searchPages.map(page => {
|
||||
const counts = termCountsFor(page);
|
||||
const terms = [...counts.entries()];
|
||||
|
||||
return {
|
||||
key: pageSearchKey(page),
|
||||
terms,
|
||||
length: terms.reduce((sum, [, count]) => sum + count, 0),
|
||||
};
|
||||
});
|
||||
const documentFrequency = new Map<string, number>();
|
||||
|
||||
for (const entry of entries) {
|
||||
for (const [term] of entry.terms) {
|
||||
documentFrequency.set(term, (documentFrequency.get(term) ?? 0) + 1);
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
averageLength:
|
||||
entries.reduce((sum, entry) => sum + entry.length, 0) / Math.max(entries.length, 1),
|
||||
documentFrequency: [...documentFrequency.entries()],
|
||||
entries,
|
||||
};
|
||||
}
|
||||
|
||||
const pages: BundlePage[] = [];
|
||||
for (const collection of COLLECTIONS) {
|
||||
const dir = join(CONTENT, collection);
|
||||
let entries: string[];
|
||||
try {
|
||||
entries = readdirSync(dir, { recursive: true }) as string[];
|
||||
} catch {
|
||||
continue;
|
||||
}
|
||||
for (const entry of entries) {
|
||||
if (!/\.mdx?$/.test(entry)) continue;
|
||||
const absPath = join(dir, entry);
|
||||
const url = urlFromContentPath(absPath);
|
||||
if (!url) continue;
|
||||
const raw = readFileSync(absPath, 'utf8');
|
||||
const { title, description, legacy } = parseFrontmatter(raw);
|
||||
pages.push({
|
||||
collection,
|
||||
url,
|
||||
title: title || url,
|
||||
description,
|
||||
legacy: legacy || isLegacyUrl(url),
|
||||
markdown: toCleanMarkdown(raw),
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
let toolkits: BundleToolkit[] = [];
|
||||
try {
|
||||
const parsed = JSON.parse(readFileSync(join(ROOT, 'public', 'data', 'toolkits.json'), 'utf8'));
|
||||
const list = Array.isArray(parsed) ? parsed : (parsed.toolkits ?? parsed.items ?? []);
|
||||
toolkits = list
|
||||
.filter((tk: { slug?: string }) => Boolean(tk?.slug))
|
||||
.map((tk: BundleToolkit) => ({
|
||||
slug: tk.slug,
|
||||
name: tk.name,
|
||||
description: tk.description,
|
||||
category: tk.category,
|
||||
authSchemes: tk.authSchemes,
|
||||
}));
|
||||
} catch {
|
||||
// no catalog available
|
||||
}
|
||||
|
||||
const contentSearchPages = pages.map(page =>
|
||||
makeSearchPage({
|
||||
collection: page.collection,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
url: page.url,
|
||||
body: page.markdown,
|
||||
})
|
||||
);
|
||||
const knowledgePages = loadKnowledgePages();
|
||||
pages.push(...knowledgePages);
|
||||
const knowledgeSearchPages = knowledgePages.map(page =>
|
||||
makeSearchPage({
|
||||
collection: page.collection,
|
||||
title: page.title,
|
||||
description: page.description,
|
||||
url: page.url,
|
||||
body: page.markdown,
|
||||
})
|
||||
);
|
||||
const toolkitSearchPages = toolkits.map(searchPageForToolkit);
|
||||
const search = buildSearchIndex([
|
||||
...contentSearchPages,
|
||||
...knowledgeSearchPages,
|
||||
...toolkitSearchPages,
|
||||
]);
|
||||
|
||||
// Emit a .ts module (not .json): eve's discovery scans agent/lib/ and only
|
||||
// accepts authored modules there, so a raw .json fails `eve build`. We embed the
|
||||
// snapshot as a JSON string parsed at runtime — a plain string literal keeps the
|
||||
// source fast for tsc and free of escaping pitfalls, and lets eve bundle it.
|
||||
const json = JSON.stringify({ pages, toolkits, search });
|
||||
const ts =
|
||||
`/* eslint-disable */\n` +
|
||||
`// Auto-generated by scripts/build-agent-index.ts — do not edit by hand.\n` +
|
||||
`// Gitignored snapshot of content/ + the toolkit catalog, imported by\n` +
|
||||
`// lib/docs.ts so eve bundles it into the deployed service (agent/ but not content/).\n` +
|
||||
`export default JSON.parse(\n ${JSON.stringify(json)},\n) as unknown;\n`;
|
||||
const out = join(ROOT, 'agent', 'lib', 'docs-index.ts');
|
||||
writeFileSync(out, ts);
|
||||
console.log(
|
||||
`[build-agent-index] wrote ${pages.length} pages + ${toolkits.length} toolkits + ${search.entries.length} BM25 rows (${Math.round(json.length / 1024)} KB) -> ${out}`
|
||||
);
|
||||
@@ -0,0 +1,413 @@
|
||||
/**
|
||||
* docs-graph.ts — content link-graph generator + connectivity checker.
|
||||
*
|
||||
* Builds a directed graph of the docs where nodes are pages and edges are
|
||||
* internal `/docs/...` links in page bodies (markdown links + `href=`).
|
||||
*
|
||||
* It then checks the *topology* (not broken links — that's validate-links.ts):
|
||||
* 1. Weakly-connected components. The biggest is "the main graph". Anything
|
||||
* else is an island — a page (or a little 2-/3-loop) that doesn't mainline
|
||||
* back into the main graph through content links.
|
||||
* 2. Reachability from `index` — pages you can't click your way to starting
|
||||
* from the homepage.
|
||||
* 3. Short directed cycles (A->B->A, A->B->C->A) that are *isolated* (their
|
||||
* own tiny component). Cycles embedded in the main graph are fine.
|
||||
*
|
||||
* Usage (from docs/):
|
||||
* bun run scripts/docs-graph.ts # report for content/docs
|
||||
* bun run scripts/docs-graph.ts --html # also write docs-graph.html
|
||||
* bun run scripts/docs-graph.ts --dot # also write docs-graph.dot
|
||||
* bun run scripts/docs-graph.ts --root content/docs --json
|
||||
*/
|
||||
|
||||
import { readFile, writeFile } from 'node:fs/promises';
|
||||
import { glob } from 'node:fs/promises';
|
||||
import { relative, resolve } from 'node:path';
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Config / args
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const args = process.argv.slice(2);
|
||||
const flag = (name: string) => args.includes(`--${name}`);
|
||||
const opt = (name: string, fallback: string) => {
|
||||
const i = args.indexOf(`--${name}`);
|
||||
return i >= 0 && args[i + 1] ? args[i + 1] : fallback;
|
||||
};
|
||||
|
||||
const ROOT = opt('root', 'content/docs'); // dir to scan, relative to cwd
|
||||
const URL_PREFIX = opt('prefix', '/docs'); // route prefix these files serve at
|
||||
const MAX_ISLAND = Number(opt('max-island', '4')); // components <= this are "small"
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Types
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
interface Node {
|
||||
/** route relative to URL_PREFIX, e.g. "providers/openai" ("" for index) */
|
||||
id: string;
|
||||
/** display label */
|
||||
label: string;
|
||||
file: string;
|
||||
out: Set<string>; // internal doc targets (node ids)
|
||||
in: Set<string>;
|
||||
/** links to other sections (/toolkits, /reference, ...) — keeps a page from
|
||||
* looking orphaned when its only links point outside docs */
|
||||
crossOut: string[];
|
||||
/** links to /docs/<x> that resolve to no known page (dangling) */
|
||||
dangling: string[];
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Build nodes
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
function routeFromFile(file: string): string {
|
||||
// file is relative to ROOT, e.g. "providers/openai.mdx" or "providers/index.mdx"
|
||||
let r = file.replace(/\.mdx$/, '');
|
||||
r = r.replace(/\/index$/, '');
|
||||
if (r === 'index') r = '';
|
||||
return r;
|
||||
}
|
||||
|
||||
const cwd = process.cwd();
|
||||
const rootAbs = resolve(cwd, ROOT);
|
||||
|
||||
const nodes = new Map<string, Node>();
|
||||
const filesByRoute = new Map<string, string>();
|
||||
|
||||
for await (const abs of glob(`${rootAbs}/**/*.mdx`)) {
|
||||
const rel = relative(rootAbs, abs);
|
||||
const id = routeFromFile(rel);
|
||||
nodes.set(id, {
|
||||
id,
|
||||
label: id === '' ? 'index' : id,
|
||||
file: `${ROOT}/${rel}`,
|
||||
out: new Set(),
|
||||
in: new Set(),
|
||||
crossOut: [],
|
||||
dangling: [],
|
||||
});
|
||||
filesByRoute.set(id, abs);
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Extract edges
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
// Matches markdown links `](/path...)`, href="/path", href={'/path'} etc.
|
||||
const LINK_RE = /(?:\]\(|href\s*=\s*["'{]?)(\/[A-Za-z0-9/_\-.#?=&]+)/g;
|
||||
|
||||
function normalizeDocTarget(raw: string): string | null {
|
||||
// strip anchor + query, trailing slash
|
||||
let t = raw.split('#')[0].split('?')[0].replace(/\/+$/, '');
|
||||
if (!t.startsWith(`${URL_PREFIX}`)) return null;
|
||||
t = t.slice(URL_PREFIX.length).replace(/^\//, ''); // -> route id
|
||||
return t;
|
||||
}
|
||||
|
||||
for (const node of nodes.values()) {
|
||||
const content = await readFile(filesByRoute.get(node.id)!, 'utf8');
|
||||
// strip fenced code blocks so code samples don't create edges
|
||||
const body = content.replace(/```[\s\S]*?```/g, '');
|
||||
for (const m of body.matchAll(LINK_RE)) {
|
||||
const href = m[1];
|
||||
if (href.startsWith(`${URL_PREFIX}/`) || href === URL_PREFIX) {
|
||||
const target = normalizeDocTarget(href);
|
||||
if (target === null) continue;
|
||||
if (target === node.id) continue; // self-anchor link
|
||||
if (nodes.has(target)) {
|
||||
node.out.add(target);
|
||||
nodes.get(target)!.in.add(node.id);
|
||||
} else {
|
||||
node.dangling.push(href);
|
||||
}
|
||||
} else {
|
||||
// cross-section internal link (/toolkits, /reference, /examples, ...)
|
||||
node.crossOut.push(href.split('#')[0]);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Graph algorithms
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const ids = [...nodes.keys()];
|
||||
|
||||
// Weakly-connected components (treat edges as undirected)
|
||||
function weaklyConnectedComponents(): string[][] {
|
||||
const seen = new Set<string>();
|
||||
const comps: string[][] = [];
|
||||
for (const start of ids) {
|
||||
if (seen.has(start)) continue;
|
||||
const stack = [start];
|
||||
const comp: string[] = [];
|
||||
seen.add(start);
|
||||
while (stack.length) {
|
||||
const n = stack.pop()!;
|
||||
comp.push(n);
|
||||
const node = nodes.get(n)!;
|
||||
for (const nb of [...node.out, ...node.in]) {
|
||||
if (!seen.has(nb)) {
|
||||
seen.add(nb);
|
||||
stack.push(nb);
|
||||
}
|
||||
}
|
||||
}
|
||||
comps.push(comp);
|
||||
}
|
||||
return comps.sort((a, b) => b.length - a.length);
|
||||
}
|
||||
|
||||
// Reachability from a root over directed out-edges
|
||||
function reachableFrom(root: string): Set<string> {
|
||||
const seen = new Set<string>([root]);
|
||||
const stack = [root];
|
||||
while (stack.length) {
|
||||
const n = stack.pop()!;
|
||||
for (const nb of nodes.get(n)!.out) {
|
||||
if (!seen.has(nb)) {
|
||||
seen.add(nb);
|
||||
stack.push(nb);
|
||||
}
|
||||
}
|
||||
}
|
||||
return seen;
|
||||
}
|
||||
|
||||
// Directed cycles up to length `maxLen` (deduped by canonical rotation)
|
||||
function shortCycles(maxLen: number): string[][] {
|
||||
const found = new Map<string, string[]>();
|
||||
const key = (cyc: string[]) => {
|
||||
const rots = cyc.map((_, i) => [...cyc.slice(i), ...cyc.slice(0, i)].join('>'));
|
||||
return rots.sort()[0];
|
||||
};
|
||||
const dfs = (start: string, path: string[]) => {
|
||||
const last = path[path.length - 1];
|
||||
for (const nb of nodes.get(last)!.out) {
|
||||
if (nb === start && path.length >= 2) {
|
||||
const cyc = [...path];
|
||||
found.set(key(cyc), cyc);
|
||||
} else if (path.length < maxLen && !path.includes(nb) && nb > start) {
|
||||
// nb > start keeps each cycle anchored at its smallest node
|
||||
dfs(start, [...path, nb]);
|
||||
}
|
||||
}
|
||||
};
|
||||
for (const id of ids) dfs(id, [id]);
|
||||
return [...found.values()];
|
||||
}
|
||||
|
||||
const components = weaklyConnectedComponents();
|
||||
const main = new Set(components[0] ?? []);
|
||||
const islands = components.slice(1);
|
||||
const reachFromIndex = nodes.has('') ? reachableFrom('') : new Set<string>();
|
||||
const cycles = shortCycles(3);
|
||||
|
||||
// A short cycle is "concerning" only if its nodes are NOT part of the main
|
||||
// component (i.e. the loop is itself an island) — embedded loops are fine.
|
||||
const isolatedCycles = cycles.filter((c) => c.some((n) => !main.has(n)));
|
||||
|
||||
// Degree + hubs. A "hub" is a high-degree page (lots of things link to/from it).
|
||||
const degree = (id: string) => nodes.get(id)!.in.size + nodes.get(id)!.out.size;
|
||||
const degrees = ids.map(degree).sort((a, b) => a - b);
|
||||
// top ~15% by degree, with a floor so tiny graphs don't flag everything
|
||||
const HUB_DEGREE = Math.max(
|
||||
Number(opt('hub-degree', '0')) || 0,
|
||||
degrees[Math.floor(degrees.length * 0.85)] ?? 0,
|
||||
6,
|
||||
);
|
||||
const isHub = (id: string) => degree(id) >= HUB_DEGREE;
|
||||
|
||||
// Mutual 2-cycles (A->B and B->A) — an immediate circle-back. Benign between
|
||||
// sibling leaf pages, but a smell between two hubs: flow ping-pongs at the top
|
||||
// of the graph instead of moving forward. We flag only the hub<->hub ones.
|
||||
const mutual2: Array<[string, string]> = [];
|
||||
const seenPair = new Set<string>();
|
||||
for (const id of ids) {
|
||||
for (const t of nodes.get(id)!.out) {
|
||||
if (nodes.get(t)!.out.has(id)) {
|
||||
const key = [id, t].sort().join('|');
|
||||
if (!seenPair.has(key)) {
|
||||
seenPair.add(key);
|
||||
mutual2.push([id, t].sort() as [string, string]);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
const hubLoops = mutual2
|
||||
.filter(([a, b]) => isHub(a) && isHub(b))
|
||||
.sort((x, y) => degree(y[0]) + degree(y[1]) - degree(x[0]) - degree(x[1]));
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Report
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
const totalEdges = ids.reduce((a, id) => a + nodes.get(id)!.out.size, 0);
|
||||
const orphans = ids.filter(
|
||||
(id) => nodes.get(id)!.in.size === 0 && nodes.get(id)!.out.size === 0,
|
||||
);
|
||||
const noInbound = ids.filter(
|
||||
(id) => id !== '' && nodes.get(id)!.in.size === 0,
|
||||
);
|
||||
const unreachable = ids.filter((id) => !reachFromIndex.has(id));
|
||||
|
||||
const C = {
|
||||
red: (s: string) => `\x1b[31m${s}\x1b[0m`,
|
||||
yellow: (s: string) => `\x1b[33m${s}\x1b[0m`,
|
||||
green: (s: string) => `\x1b[32m${s}\x1b[0m`,
|
||||
dim: (s: string) => `\x1b[2m${s}\x1b[0m`,
|
||||
bold: (s: string) => `\x1b[1m${s}\x1b[0m`,
|
||||
};
|
||||
|
||||
console.log(C.bold(`\nDocs link graph — ${ROOT}`));
|
||||
console.log(
|
||||
`${nodes.size} pages · ${totalEdges} internal links · ${components.length} component(s)\n`,
|
||||
);
|
||||
|
||||
console.log(C.bold('Main graph'));
|
||||
console.log(` ${main.size}/${nodes.size} pages connected into the main component\n`);
|
||||
|
||||
if (islands.length) {
|
||||
console.log(C.red(C.bold(`Islands (${islands.length}) — not connected to the main graph`)));
|
||||
for (const comp of islands) {
|
||||
const kind =
|
||||
comp.length === 1 ? 'orphan page' : `${comp.length}-page loop/cluster`;
|
||||
console.log(` ${C.red('●')} ${kind}`);
|
||||
for (const n of comp) {
|
||||
const node = nodes.get(n)!;
|
||||
const outNote = node.crossOut.length
|
||||
? C.dim(` (links out to ${[...new Set(node.crossOut)].slice(0, 3).join(', ')})`)
|
||||
: node.out.size || node.in.size
|
||||
? ''
|
||||
: C.dim(' (no links at all)');
|
||||
console.log(` ${node.label || 'index'}${outNote}`);
|
||||
}
|
||||
}
|
||||
console.log();
|
||||
} else {
|
||||
console.log(C.green('No islands — every page connects into the main graph.\n'));
|
||||
}
|
||||
|
||||
if (hubLoops.length) {
|
||||
console.log(C.red(C.bold(`Hub ping-pong loops (${hubLoops.length})`)));
|
||||
console.log(C.dim(' (two high-traffic pages that only circle back to each other — flow should move forward)'));
|
||||
for (const [a, b] of hubLoops) {
|
||||
console.log(` ${C.red('⇄')} ${a || 'index'} ⇄ ${b || 'index'} ${C.dim(`(deg ${degree(a)} / ${degree(b)})`)}`);
|
||||
}
|
||||
console.log();
|
||||
} else {
|
||||
console.log(C.green('No hub ping-pong loops.\n'));
|
||||
}
|
||||
|
||||
if (isolatedCycles.length) {
|
||||
console.log(C.yellow(C.bold(`Isolated short cycles (${isolatedCycles.length})`)));
|
||||
console.log(C.dim(' (pages that circle straight back without joining the main graph)'));
|
||||
for (const c of isolatedCycles) {
|
||||
console.log(` ${C.yellow('↻')} ${c.map((n) => n || 'index').join(' → ')} → ${c[0] || 'index'}`);
|
||||
}
|
||||
console.log();
|
||||
}
|
||||
|
||||
// Click-reachability from index is only meaningful if index actually links out.
|
||||
// (Docs here navigate by sidebar, so an index with no body links makes every
|
||||
// page "unreachable" — that's noise, and index already shows up as an island.)
|
||||
if (nodes.has('') && nodes.get('')!.out.size === 0) {
|
||||
console.log(C.dim('Click-reachability from index: N/A (index has no outbound content links)\n'));
|
||||
} else if (unreachable.length && nodes.has('')) {
|
||||
console.log(C.yellow(C.bold(`Not reachable from index by clicking (${unreachable.length})`)));
|
||||
console.log(C.dim(' (in the sidebar, but no content link path from the homepage)'));
|
||||
for (const id of unreachable.slice(0, 40)) {
|
||||
console.log(` ${C.yellow('·')} ${id || 'index'}`);
|
||||
}
|
||||
if (unreachable.length > 40) console.log(C.dim(` … +${unreachable.length - 40} more`));
|
||||
console.log();
|
||||
}
|
||||
|
||||
if (noInbound.length) {
|
||||
console.log(C.dim(`Pages with no inbound content links (${noInbound.length}): ${noInbound.slice(0, 12).join(', ')}${noInbound.length > 12 ? ', …' : ''}\n`));
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Optional outputs
|
||||
// ---------------------------------------------------------------------------
|
||||
|
||||
if (flag('json')) {
|
||||
const out = {
|
||||
nodes: ids.map((id) => ({
|
||||
id,
|
||||
out: [...nodes.get(id)!.out],
|
||||
in: [...nodes.get(id)!.in],
|
||||
crossOut: nodes.get(id)!.crossOut,
|
||||
dangling: nodes.get(id)!.dangling,
|
||||
})),
|
||||
components,
|
||||
islands,
|
||||
isolatedCycles,
|
||||
unreachable,
|
||||
};
|
||||
await writeFile('docs-graph.json', JSON.stringify(out, null, 2));
|
||||
console.log(C.dim('Wrote docs-graph.json'));
|
||||
}
|
||||
|
||||
if (flag('dot')) {
|
||||
const compIndex = new Map<string, number>();
|
||||
components.forEach((c, i) => c.forEach((n) => compIndex.set(n, i)));
|
||||
const palette = ['#2563eb', '#dc2626', '#d97706', '#7c3aed', '#059669', '#db2777'];
|
||||
const lines = ['digraph docs {', ' rankdir=LR;', ' node [shape=box, style=rounded, fontname="Helvetica"];'];
|
||||
for (const id of ids) {
|
||||
const ci = compIndex.get(id)!;
|
||||
const color = ci === 0 ? '#94a3b8' : palette[(ci - 1) % palette.length];
|
||||
const isIsland = ci !== 0;
|
||||
lines.push(` "${id || 'index'}" [color="${color}"${isIsland ? ', penwidth=2, fontcolor="' + color + '"' : ''}];`);
|
||||
}
|
||||
for (const id of ids) {
|
||||
for (const t of nodes.get(id)!.out) {
|
||||
lines.push(` "${id || 'index'}" -> "${t || 'index'}";`);
|
||||
}
|
||||
}
|
||||
lines.push('}');
|
||||
await writeFile('docs-graph.dot', lines.join('\n'));
|
||||
console.log(C.dim('Wrote docs-graph.dot (render: dot -Tsvg docs-graph.dot -o docs-graph.svg)'));
|
||||
}
|
||||
|
||||
if (flag('html')) {
|
||||
const compIndex = new Map<string, number>();
|
||||
components.forEach((c, i) => c.forEach((n) => compIndex.set(n, i)));
|
||||
const data = {
|
||||
nodes: ids.map((id) => ({
|
||||
id: id || 'index',
|
||||
comp: compIndex.get(id)!,
|
||||
island: compIndex.get(id)! !== 0,
|
||||
degree: nodes.get(id)!.in.size + nodes.get(id)!.out.size,
|
||||
})),
|
||||
edges: ids.flatMap((id) =>
|
||||
[...nodes.get(id)!.out].map((t) => ({ from: id || 'index', to: t || 'index' })),
|
||||
),
|
||||
};
|
||||
const html = `<!doctype html><meta charset=utf8><title>Docs link graph</title>
|
||||
<style>html,body{margin:0;height:100%;background:#0b0f17;color:#e5e7eb;font-family:system-ui}
|
||||
#net{height:100vh}#legend{position:fixed;top:12px;left:12px;background:#111827cc;padding:10px 14px;border-radius:8px;font-size:13px;line-height:1.6}</style>
|
||||
<div id=legend><b>Docs link graph</b><br><span style=color:#94a3b8>●</span> main graph <span style=color:#dc2626>●</span> island (disconnected)<br>node size = link count</div>
|
||||
<div id=net></div>
|
||||
<script src="https://unpkg.com/vis-network/standalone/umd/vis-network.min.js"></script>
|
||||
<script>
|
||||
const D=${JSON.stringify(data)};
|
||||
const palette=['#94a3b8','#dc2626','#d97706','#7c3aed','#059669','#db2777','#2563eb'];
|
||||
const nodes=new vis.DataSet(D.nodes.map(n=>({id:n.id,label:n.id,value:n.degree+1,
|
||||
color:{background:n.island?palette[1+(n.comp-1)%6]:'#334155',border:n.island?palette[1+(n.comp-1)%6]:'#64748b'},
|
||||
font:{color:n.island?'#fca5a5':'#cbd5e1'}})));
|
||||
const edges=new vis.DataSet(D.edges.map(e=>({from:e.from,to:e.to,arrows:'to',color:{color:'#33415588'}})));
|
||||
new vis.Network(document.getElementById('net'),{nodes,edges},{
|
||||
physics:{stabilization:true,barnesHut:{gravitationalConstant:-8000,springLength:120}},
|
||||
nodes:{shape:'dot',scaling:{min:6,max:40}},interaction:{hover:true}});
|
||||
</script>`;
|
||||
await writeFile('docs-graph.html', html);
|
||||
console.log(C.dim('Wrote docs-graph.html (open in a browser)'));
|
||||
}
|
||||
|
||||
// exit non-zero if there are islands, so this can gate CI
|
||||
if (flag('check') && (islands.length || isolatedCycles.length || hubLoops.length)) {
|
||||
process.exit(1);
|
||||
}
|
||||
@@ -0,0 +1,110 @@
|
||||
#!/usr/bin/env bun
|
||||
|
||||
import { spawnSync } from 'node:child_process';
|
||||
|
||||
const DEFAULT_LOCAL_FLOWS = ['gateway', 'mercury'] as const;
|
||||
const EVE_BIN = process.env.EVE_BIN ?? './node_modules/.bin/eve';
|
||||
const extraArgs = process.argv.slice(2);
|
||||
|
||||
type EvalRun = {
|
||||
name: string;
|
||||
env?: Record<string, string>;
|
||||
url?: string;
|
||||
};
|
||||
|
||||
const parseList = (value: string | undefined): string[] =>
|
||||
value
|
||||
?.split(',')
|
||||
.map(item => item.trim())
|
||||
.filter(Boolean) ?? [];
|
||||
|
||||
const parseRemoteTargets = (value: string | undefined): EvalRun[] =>
|
||||
parseList(value).map(entry => {
|
||||
const separator = entry.indexOf('=');
|
||||
if (separator === -1) {
|
||||
throw new Error(
|
||||
`Invalid DOCS_AGENT_EVAL_TARGETS entry "${entry}". Use name=https://deployment.example.`
|
||||
);
|
||||
}
|
||||
|
||||
const name = entry.slice(0, separator).trim();
|
||||
const url = entry.slice(separator + 1).trim();
|
||||
|
||||
if (!name || !url) {
|
||||
throw new Error(
|
||||
`Invalid DOCS_AGENT_EVAL_TARGETS entry "${entry}". Both name and URL are required.`
|
||||
);
|
||||
}
|
||||
|
||||
return { name, url };
|
||||
});
|
||||
|
||||
const buildRuns = (): EvalRun[] => {
|
||||
const remoteTargets = parseRemoteTargets(process.env.DOCS_AGENT_EVAL_TARGETS);
|
||||
|
||||
if (remoteTargets.length > 0) {
|
||||
return remoteTargets;
|
||||
}
|
||||
|
||||
const flows = parseList(process.env.DOCS_AGENT_EVAL_FLOWS);
|
||||
const selectedFlows = flows.length > 0 ? flows : [...DEFAULT_LOCAL_FLOWS];
|
||||
|
||||
return selectedFlows.map(flow => ({
|
||||
name: flow,
|
||||
env: { DOCS_AGENT_MODEL_FLOW: flow },
|
||||
}));
|
||||
};
|
||||
|
||||
const warnForMissingCredentials = (run: EvalRun) => {
|
||||
const flow = run.env?.DOCS_AGENT_MODEL_FLOW;
|
||||
|
||||
if (flow === 'mercury' && !process.env.INCEPTION_API_KEY) {
|
||||
console.warn(
|
||||
'[eval-agent-flows] INCEPTION_API_KEY is not set; Mercury evals will fail or skip model calls.'
|
||||
);
|
||||
}
|
||||
|
||||
if (flow === 'gateway' && !process.env.AI_GATEWAY_API_KEY && !process.env.VERCEL_OIDC_TOKEN) {
|
||||
console.warn(
|
||||
'[eval-agent-flows] AI_GATEWAY_API_KEY/VERCEL_OIDC_TOKEN is not set; gateway evals will fail or skip model calls.'
|
||||
);
|
||||
}
|
||||
};
|
||||
|
||||
const runEval = (run: EvalRun) => {
|
||||
const args = ['eval', 'docs-agent', '--skip-report'];
|
||||
|
||||
if (run.url) {
|
||||
args.push('--url', run.url);
|
||||
}
|
||||
|
||||
args.push(...extraArgs);
|
||||
|
||||
console.log(`\n## ${run.url ? 'Remote target' : 'Local model flow'}: ${run.name}`);
|
||||
console.log(`$ ${EVE_BIN} ${args.join(' ')}`);
|
||||
warnForMissingCredentials(run);
|
||||
|
||||
return (
|
||||
spawnSync(EVE_BIN, args, {
|
||||
env: { ...process.env, ...run.env },
|
||||
stdio: 'inherit',
|
||||
}).status ?? 1
|
||||
);
|
||||
};
|
||||
|
||||
const runs = buildRuns();
|
||||
let failed = 0;
|
||||
|
||||
for (const run of runs) {
|
||||
const status = runEval(run);
|
||||
if (status !== 0) {
|
||||
failed += 1;
|
||||
}
|
||||
}
|
||||
|
||||
if (failed > 0) {
|
||||
console.error(`\n${failed}/${runs.length} docs-agent eval run(s) failed.`);
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
console.log(`\nAll ${runs.length} docs-agent eval run(s) passed.`);
|
||||
@@ -0,0 +1,244 @@
|
||||
/**
|
||||
* Fetches and filters the OpenAPI specs for fumadocs.
|
||||
* Outputs two separate spec files:
|
||||
* - public/openapi.json (v3.1 — latest, clean operationIds)
|
||||
* - public/openapi-v3.json (v3.0)
|
||||
*
|
||||
* Run: bun run scripts/fetch-openapi.mjs
|
||||
*/
|
||||
|
||||
import { writeFileSync } from 'node:fs';
|
||||
import { fileURLToPath } from 'node:url';
|
||||
import { dirname, join } from 'node:path';
|
||||
|
||||
const OPENAPI_V3_URL = process.env.OPENAPI_SPEC_URL || 'https://backend.composio.dev/api/v3/openapi.json';
|
||||
const OPENAPI_V31_URL = process.env.OPENAPI_V31_SPEC_URL || 'https://backend.composio.dev/api/v3.1/openapi.json';
|
||||
|
||||
// Tags to ignore (internal/admin)
|
||||
const IGNORED_TAGS = [
|
||||
'CLI',
|
||||
'Admin',
|
||||
'Profiling',
|
||||
'User',
|
||||
'x-internal',
|
||||
];
|
||||
|
||||
async function fetchSpec(url) {
|
||||
console.log(`Fetching OpenAPI spec from ${url}...`);
|
||||
const response = await fetch(url);
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch ${url}: ${response.status}`);
|
||||
}
|
||||
return response.json();
|
||||
}
|
||||
|
||||
/**
|
||||
* Filter paths: remove ignored/internal tags, keep first tag only.
|
||||
*/
|
||||
function filterPaths(paths) {
|
||||
const filteredPaths = {};
|
||||
let removedCount = 0;
|
||||
|
||||
for (const [path, methods] of Object.entries(paths)) {
|
||||
const filteredMethods = {};
|
||||
|
||||
for (const [method, operation] of Object.entries(methods)) {
|
||||
const tags = operation.tags || [];
|
||||
const hasValidTag = tags.some(tag => !IGNORED_TAGS.includes(tag));
|
||||
|
||||
if (!hasValidTag && tags.length > 0) {
|
||||
removedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (operation['x-internal'] === true || tags.includes('x-internal')) {
|
||||
removedCount++;
|
||||
continue;
|
||||
}
|
||||
|
||||
if (tags.length > 1) {
|
||||
operation.tags = [tags[0]];
|
||||
}
|
||||
|
||||
filteredMethods[method] = operation;
|
||||
}
|
||||
|
||||
if (Object.keys(filteredMethods).length > 0) {
|
||||
filteredPaths[path] = filteredMethods;
|
||||
}
|
||||
}
|
||||
|
||||
return { filteredPaths, removedCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* Strip version prefixes from operationIds (e.g. getV3_1Tools → getTools).
|
||||
*/
|
||||
function cleanOperationIds(paths) {
|
||||
for (const methods of Object.values(paths)) {
|
||||
for (const operation of Object.values(methods)) {
|
||||
if (operation.operationId) {
|
||||
// Remove V3_1, V3_0, etc. prefixes from operationId
|
||||
operation.operationId = operation.operationId.replace(/V\d+_\d+/g, '');
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Post-process a spec: remove CookieAuth, normalize unions, fix nullable.
|
||||
*/
|
||||
function postProcessSpec(spec) {
|
||||
// Pin the server to production. The published docs must always show the
|
||||
// production base URL in their curl examples, regardless of which environment
|
||||
// the source spec was fetched from (a staging fetch would otherwise bake a
|
||||
// staging server URL into the committed reference).
|
||||
spec.servers = [
|
||||
{
|
||||
url: 'https://backend.composio.dev',
|
||||
description: 'PRODUCTION API',
|
||||
},
|
||||
];
|
||||
|
||||
// Filter tags list
|
||||
if (spec.tags) {
|
||||
spec.tags = spec.tags.filter(tag => !IGNORED_TAGS.includes(tag.name));
|
||||
}
|
||||
|
||||
// Remove CookieAuth from security schemes
|
||||
if (spec.components?.securitySchemes?.CookieAuth) {
|
||||
delete spec.components.securitySchemes.CookieAuth;
|
||||
}
|
||||
|
||||
// Remove CookieAuth from all endpoint security arrays
|
||||
for (const methods of Object.values(spec.paths)) {
|
||||
for (const operation of Object.values(methods)) {
|
||||
if (operation.security) {
|
||||
operation.security = operation.security.filter(sec => !('CookieAuth' in sec));
|
||||
if (operation.security.length === 0) {
|
||||
delete operation.security;
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
// Normalize overly complex anyOf/oneOf schemas
|
||||
const mergePropertySchemas = (existing, incoming) => {
|
||||
if (!existing) return JSON.parse(JSON.stringify(incoming));
|
||||
const merged = JSON.parse(JSON.stringify(existing));
|
||||
if (existing.enum && incoming.enum) {
|
||||
merged.enum = [...new Set([...existing.enum, ...incoming.enum])];
|
||||
}
|
||||
if (existing.properties && incoming.properties) {
|
||||
merged.properties = { ...existing.properties };
|
||||
for (const [key, val] of Object.entries(incoming.properties)) {
|
||||
merged.properties[key] = mergePropertySchemas(merged.properties[key], val);
|
||||
}
|
||||
}
|
||||
return merged;
|
||||
};
|
||||
|
||||
const normalizeUnionSchemas = (obj) => {
|
||||
if (!obj || typeof obj !== 'object') return;
|
||||
for (const unionKey of ['anyOf', 'oneOf']) {
|
||||
if (obj[unionKey] && Array.isArray(obj[unionKey]) && obj[unionKey].length > 5) {
|
||||
const objectSchemas = obj[unionKey].filter(s => s.type === 'object' && s.properties);
|
||||
if (objectSchemas.length > 5 && objectSchemas.length >= obj[unionKey].length * 0.8) {
|
||||
const mergedProperties = {};
|
||||
const allRequired = new Set();
|
||||
for (const schema of objectSchemas) {
|
||||
for (const [propName, propSchema] of Object.entries(schema.properties || {})) {
|
||||
mergedProperties[propName] = mergePropertySchemas(mergedProperties[propName], propSchema);
|
||||
}
|
||||
if (schema.required) {
|
||||
for (const req of schema.required) allRequired.add(req);
|
||||
}
|
||||
}
|
||||
const universallyRequired = [...allRequired].filter(req =>
|
||||
objectSchemas.every(s => s.required && s.required.includes(req))
|
||||
);
|
||||
delete obj[unionKey];
|
||||
obj.type = 'object';
|
||||
obj.properties = mergedProperties;
|
||||
if (universallyRequired.length > 0) obj.required = universallyRequired;
|
||||
obj.additionalProperties = true;
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const val of Object.values(obj)) {
|
||||
if (Array.isArray(val)) val.forEach(item => normalizeUnionSchemas(item));
|
||||
else normalizeUnionSchemas(val);
|
||||
}
|
||||
};
|
||||
normalizeUnionSchemas(spec);
|
||||
|
||||
// Fix invalid OpenAPI 3.0: "nullable: true" without "type"
|
||||
const fixNullableWithoutType = (obj, parentKey = '') => {
|
||||
if (!obj || typeof obj !== 'object') return;
|
||||
if (obj.nullable === true && !obj.type && !obj.$ref && !obj.oneOf && !obj.anyOf && !obj.allOf) {
|
||||
if (parentKey === 'additionalProperties') {
|
||||
delete obj.nullable;
|
||||
} else if (obj.example && typeof obj.example === 'object' && !Array.isArray(obj.example)) {
|
||||
obj.type = 'object';
|
||||
} else if (obj.example && Array.isArray(obj.example)) {
|
||||
obj.type = 'array';
|
||||
} else {
|
||||
obj.type = 'object';
|
||||
}
|
||||
}
|
||||
for (const [key, val] of Object.entries(obj)) {
|
||||
if (Array.isArray(val)) val.forEach(item => fixNullableWithoutType(item, key));
|
||||
else fixNullableWithoutType(val, key);
|
||||
}
|
||||
};
|
||||
fixNullableWithoutType(spec);
|
||||
}
|
||||
|
||||
async function fetchAndFilterSpec() {
|
||||
// Fetch both specs in parallel
|
||||
const [v3Raw, v31Raw] = await Promise.all([
|
||||
fetchSpec(OPENAPI_V3_URL),
|
||||
fetchSpec(OPENAPI_V31_URL),
|
||||
]);
|
||||
|
||||
// --- v3.1 spec (latest, default) ---
|
||||
const v31Spec = JSON.parse(JSON.stringify(v31Raw));
|
||||
const v31Filtered = filterPaths(v31Spec.paths);
|
||||
v31Spec.paths = v31Filtered.filteredPaths;
|
||||
// Clean operationIds: getV3_1Tools → getTools (so URLs are clean)
|
||||
cleanOperationIds(v31Spec.paths);
|
||||
// Annotate all operations with version
|
||||
for (const methods of Object.values(v31Spec.paths)) {
|
||||
for (const op of Object.values(methods)) {
|
||||
op['x-api-version'] = '3.1';
|
||||
}
|
||||
}
|
||||
postProcessSpec(v31Spec);
|
||||
console.log(`v3.1: ${Object.keys(v31Spec.paths).length} paths`);
|
||||
|
||||
// --- v3.0 spec ---
|
||||
const v3Spec = JSON.parse(JSON.stringify(v3Raw));
|
||||
const v3Filtered = filterPaths(v3Spec.paths);
|
||||
v3Spec.paths = v3Filtered.filteredPaths;
|
||||
cleanOperationIds(v3Spec.paths);
|
||||
for (const methods of Object.values(v3Spec.paths)) {
|
||||
for (const op of Object.values(methods)) {
|
||||
op['x-api-version'] = '3.0';
|
||||
}
|
||||
}
|
||||
postProcessSpec(v3Spec);
|
||||
console.log(`v3.0: ${Object.keys(v3Spec.paths).length} paths`);
|
||||
|
||||
// Write both spec files
|
||||
const __dirname = dirname(fileURLToPath(import.meta.url));
|
||||
|
||||
const v31Path = join(__dirname, '../public/openapi.json');
|
||||
writeFileSync(v31Path, JSON.stringify(v31Spec, null, 2));
|
||||
console.log(`Written v3.1 spec to ${v31Path}`);
|
||||
|
||||
const v3Path = join(__dirname, '../public/openapi-v3.json');
|
||||
writeFileSync(v3Path, JSON.stringify(v3Spec, null, 2));
|
||||
console.log(`Written v3.0 spec to ${v3Path}`);
|
||||
}
|
||||
|
||||
fetchAndFilterSpec().catch(console.error);
|
||||
@@ -0,0 +1,140 @@
|
||||
/**
|
||||
* Shared fetch helper for docs generation scripts.
|
||||
*
|
||||
* Wraps the global `fetch` with rate-limit-aware retry/backoff:
|
||||
* - Retries `429 Too Many Requests` and transient 5xx responses.
|
||||
* - Honors the `Retry-After` header (delta-seconds or HTTP-date) when present.
|
||||
* - Falls back to exponential backoff with full jitter.
|
||||
* - Logs the endpoint and retry count so CI logs explain the delay.
|
||||
* - Caps attempts so CI still fails fast when the backend is genuinely down —
|
||||
* the final failing `Response` is returned to the caller, which keeps existing
|
||||
* `if (!response.ok)` handling intact.
|
||||
*
|
||||
* Network-level errors thrown by `fetch` (DNS, connection reset, …) are retried
|
||||
* the same way and re-thrown once attempts are exhausted.
|
||||
*
|
||||
* Why this exists: `generate-toolkits.ts` issues ~6500 requests per run (3 per
|
||||
* toolkit across a ~2.1k catalog), which exceeds the staging limit of 2000
|
||||
* requests/minute. Without backoff the run
|
||||
* fails with `429`, and `generate-meta-tools.ts` (which runs next) inherits the
|
||||
* exhausted window. See docs/scripts/README.md.
|
||||
*
|
||||
* Deliberately hand-rolled rather than using Effect's `Schedule` (which would be
|
||||
* the idiomatic fit elsewhere): the docs package has no Effect dependency and
|
||||
* these are plain bun scripts. Effect is reserved for `ts/packages/cli`. Keep
|
||||
* this helper dependency-free.
|
||||
*/
|
||||
|
||||
const DEFAULT_RETRY_STATUSES = [429, 500, 502, 503, 504];
|
||||
|
||||
export interface FetchWithRetryOptions extends RequestInit {
|
||||
/** Maximum number of attempts, including the first. Default 6. */
|
||||
maxAttempts?: number;
|
||||
/** Base delay in ms for exponential backoff. Default 1000. */
|
||||
baseDelayMs?: number;
|
||||
/** Upper bound for a single backoff wait in ms. Default 60000. */
|
||||
maxDelayMs?: number;
|
||||
/** HTTP status codes that trigger a retry. Default [429, 500, 502, 503, 504]. */
|
||||
retryStatuses?: number[];
|
||||
}
|
||||
|
||||
function sleep(ms: number): Promise<void> {
|
||||
return new Promise((resolve) => setTimeout(resolve, ms));
|
||||
}
|
||||
|
||||
/** Parse a `Retry-After` header (delta-seconds or HTTP-date) into ms, or null. */
|
||||
function parseRetryAfter(header: string | null): number | null {
|
||||
if (!header) return null;
|
||||
|
||||
const seconds = Number(header);
|
||||
if (Number.isFinite(seconds)) {
|
||||
return Math.max(0, seconds * 1000);
|
||||
}
|
||||
|
||||
const dateMs = Date.parse(header);
|
||||
if (!Number.isNaN(dateMs)) {
|
||||
return Math.max(0, dateMs - Date.now());
|
||||
}
|
||||
|
||||
return null;
|
||||
}
|
||||
|
||||
/** Exponential backoff with full jitter. `attempt` is 1-based. */
|
||||
function backoffDelay(attempt: number, baseDelayMs: number, maxDelayMs: number): number {
|
||||
const ceiling = Math.min(maxDelayMs, baseDelayMs * 2 ** (attempt - 1));
|
||||
return Math.random() * ceiling;
|
||||
}
|
||||
|
||||
/** Collapse a URL to `origin + pathname` for tidy log lines (drops query noise). */
|
||||
function loggableEndpoint(url: string): string {
|
||||
try {
|
||||
const parsed = new URL(url);
|
||||
return `${parsed.origin}${parsed.pathname}`;
|
||||
} catch {
|
||||
return url;
|
||||
}
|
||||
}
|
||||
|
||||
export async function fetchWithRetry(
|
||||
url: string,
|
||||
options: FetchWithRetryOptions = {}
|
||||
): Promise<Response> {
|
||||
const {
|
||||
maxAttempts = 6,
|
||||
baseDelayMs = 1000,
|
||||
maxDelayMs = 60_000,
|
||||
retryStatuses = DEFAULT_RETRY_STATUSES,
|
||||
...init
|
||||
} = options;
|
||||
|
||||
const endpoint = loggableEndpoint(url);
|
||||
let lastError: unknown;
|
||||
|
||||
for (let attempt = 1; attempt <= maxAttempts; attempt++) {
|
||||
let response: Response;
|
||||
|
||||
try {
|
||||
response = await fetch(url, init);
|
||||
} catch (error) {
|
||||
lastError = error;
|
||||
if (attempt >= maxAttempts) break;
|
||||
const delay = backoffDelay(attempt, baseDelayMs, maxDelayMs);
|
||||
console.warn(
|
||||
` [fetch] ${endpoint} network error (attempt ${attempt}/${maxAttempts}), ` +
|
||||
`retrying in ${Math.round(delay)}ms: ${(error as Error).message}`
|
||||
);
|
||||
await sleep(delay);
|
||||
continue;
|
||||
}
|
||||
|
||||
// Success, or a non-retryable status: hand it back to the caller.
|
||||
if (!retryStatuses.includes(response.status)) {
|
||||
return response;
|
||||
}
|
||||
|
||||
// Out of attempts: surface the failing response so callers still fail.
|
||||
if (attempt >= maxAttempts) {
|
||||
return response;
|
||||
}
|
||||
|
||||
const retryAfterMs = parseRetryAfter(response.headers.get('retry-after'));
|
||||
// Add jitter on top of Retry-After to avoid a synchronized retry burst.
|
||||
const delay =
|
||||
retryAfterMs != null
|
||||
? retryAfterMs + Math.random() * 1000
|
||||
: backoffDelay(attempt, baseDelayMs, maxDelayMs);
|
||||
|
||||
console.warn(
|
||||
` [fetch] ${endpoint} -> ${response.status} (attempt ${attempt}/${maxAttempts}), ` +
|
||||
`retrying in ${Math.round(delay)}ms${retryAfterMs != null ? ' (Retry-After)' : ''}`
|
||||
);
|
||||
|
||||
// Drain the body so the connection can be reused.
|
||||
await response.arrayBuffer().catch(() => {});
|
||||
await sleep(delay);
|
||||
}
|
||||
|
||||
throw lastError instanceof Error
|
||||
? lastError
|
||||
: new Error(`fetchWithRetry: exhausted ${maxAttempts} attempts for ${endpoint}`);
|
||||
}
|
||||
@@ -0,0 +1,276 @@
|
||||
/**
|
||||
* Generates markdown index pages for each OpenAPI tag.
|
||||
* Reads both v3.1 and v3.0 specs and generates a table that
|
||||
* uses the ApiEndpointsTable component to switch versions dynamically.
|
||||
*
|
||||
* Run: bun scripts/generate-api-index.ts
|
||||
*/
|
||||
|
||||
import { readFileSync, writeFileSync, mkdirSync, existsSync, rmSync, readdirSync } from 'fs';
|
||||
import { join } from 'path';
|
||||
import { HIDDEN_API_TAGS } from '../lib/filter-api-version';
|
||||
|
||||
/**
|
||||
* API-reference tags hidden on our side even though the upstream OpenAPI spec
|
||||
* (from hermes) includes them. Matched by slug. We neither generate their
|
||||
* `index.mdx` overview pages nor leave stale ones behind. Shared with the
|
||||
* reference page-tree filter so both stay in sync.
|
||||
*/
|
||||
const HIDDEN_TAGS: ReadonlySet<string> = HIDDEN_API_TAGS;
|
||||
|
||||
/**
|
||||
* Display-title overrides for API-reference tags whose upstream OpenAPI tag
|
||||
* name is stale or off-brand. Keyed by tag slug.
|
||||
*/
|
||||
const TITLE_OVERRIDES: Record<string, string> = {
|
||||
'tool-router': 'Sessions (prev Tool Router)',
|
||||
};
|
||||
|
||||
interface OpenAPIOperation {
|
||||
summary?: string;
|
||||
tags?: string[];
|
||||
description?: string;
|
||||
operationId?: string;
|
||||
'x-api-version'?: string;
|
||||
}
|
||||
|
||||
interface OpenAPISpec {
|
||||
tags: Array<{ name: string; description?: string }>;
|
||||
paths: Record<string, Record<string, OpenAPIOperation>>;
|
||||
}
|
||||
|
||||
interface OperationEntry {
|
||||
summary: string;
|
||||
method: string;
|
||||
path: string;
|
||||
operationId: string;
|
||||
}
|
||||
|
||||
function slugify(text: string): string {
|
||||
return text
|
||||
.toLowerCase()
|
||||
.replace(/[^a-z0-9]+/g, '-')
|
||||
.replace(/^-|-$/g, '');
|
||||
}
|
||||
|
||||
/**
|
||||
* Optional hand-written overview for a tag, merged above the generated
|
||||
* endpoints table. Lives in `api-overviews/<tagSlug>.mdx` (outside `content/`,
|
||||
* so Fumadocs never renders it as its own page). Use this to fold a conceptual
|
||||
* guide into the API reference page instead of keeping a separate docs page.
|
||||
* Frontmatter, if present, is stripped — the generator owns the frontmatter.
|
||||
*/
|
||||
function readOverview(tagSlug: string): string | null {
|
||||
const overviewPath = join(process.cwd(), 'api-overviews', `${tagSlug}.mdx`);
|
||||
if (!existsSync(overviewPath)) return null;
|
||||
const raw = readFileSync(overviewPath, 'utf-8');
|
||||
const stripped = raw.replace(/^---\n[\s\S]*?\n---\n/, '').trim();
|
||||
return stripped.length > 0 ? stripped : null;
|
||||
}
|
||||
|
||||
function getOperationsByTag(spec: OpenAPISpec): Record<string, OperationEntry[]> {
|
||||
const tagOps: Record<string, OperationEntry[]> = {};
|
||||
|
||||
for (const tag of spec.tags) {
|
||||
tagOps[tag.name] = [];
|
||||
}
|
||||
|
||||
for (const [path, methods] of Object.entries(spec.paths)) {
|
||||
for (const [method, operation] of Object.entries(methods)) {
|
||||
if (operation.tags) {
|
||||
for (const tag of operation.tags) {
|
||||
if (!tagOps[tag]) tagOps[tag] = [];
|
||||
tagOps[tag].push({
|
||||
summary: operation.summary || `${method.toUpperCase()} ${path}`,
|
||||
method: method.toUpperCase(),
|
||||
path,
|
||||
operationId: operation.operationId || slugify(operation.summary || path),
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
return tagOps;
|
||||
}
|
||||
|
||||
function activeTagSlugs(opsByTag: Record<string, OperationEntry[]>): Set<string> {
|
||||
const active = new Set<string>();
|
||||
for (const [tagName, ops] of Object.entries(opsByTag)) {
|
||||
const tagSlug = slugify(tagName);
|
||||
if (ops.length > 0 && !HIDDEN_TAGS.has(tagSlug)) {
|
||||
active.add(tagSlug);
|
||||
}
|
||||
}
|
||||
return active;
|
||||
}
|
||||
|
||||
function removeStaleTagIndexes(baseDir: string, activeSlugs: Set<string>) {
|
||||
if (!existsSync(baseDir)) return;
|
||||
|
||||
for (const entry of readdirSync(baseDir, { withFileTypes: true })) {
|
||||
if (!entry.isDirectory()) continue;
|
||||
|
||||
const tagSlug = entry.name;
|
||||
if (activeSlugs.has(tagSlug)) continue;
|
||||
|
||||
const tagDir = join(baseDir, tagSlug);
|
||||
const indexPath = join(tagDir, 'index.mdx');
|
||||
if (existsSync(indexPath)) {
|
||||
rmSync(tagDir, { recursive: true, force: true });
|
||||
console.log(`Removed stale tag: ${tagDir}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function generateIndexPages() {
|
||||
const specV31Path = join(process.cwd(), 'public/openapi.json');
|
||||
const specV3Path = join(process.cwd(), 'public/openapi-v3.json');
|
||||
|
||||
const specV31: OpenAPISpec = JSON.parse(readFileSync(specV31Path, 'utf-8'));
|
||||
const v31Ops = getOperationsByTag(specV31);
|
||||
|
||||
let v3Ops: Record<string, OperationEntry[]> = {};
|
||||
if (existsSync(specV3Path)) {
|
||||
const specV3: OpenAPISpec = JSON.parse(readFileSync(specV3Path, 'utf-8'));
|
||||
v3Ops = getOperationsByTag(specV3);
|
||||
}
|
||||
|
||||
// Collect tag descriptions from v3.1 spec
|
||||
const tagDescriptions: Record<string, string> = {};
|
||||
for (const tag of specV31.tags) {
|
||||
tagDescriptions[tag.name] = tag.description || '';
|
||||
}
|
||||
|
||||
const outputDir = join(process.cwd(), 'content/reference/api-reference');
|
||||
|
||||
removeStaleTagIndexes(outputDir, activeTagSlugs(v31Ops));
|
||||
removeStaleTagIndexes(
|
||||
join(process.cwd(), 'content/reference/v3/api-reference'),
|
||||
activeTagSlugs(v3Ops),
|
||||
);
|
||||
|
||||
// Get all unique tag names
|
||||
const allTags = new Set([...Object.keys(v31Ops), ...Object.keys(v3Ops)]);
|
||||
|
||||
for (const tagName of allTags) {
|
||||
const ops31 = v31Ops[tagName] || [];
|
||||
const ops3 = v3Ops[tagName] || [];
|
||||
const tagSlug = slugify(tagName);
|
||||
|
||||
// Intentionally-hidden tag — skip generation and delete any existing index.mdx
|
||||
// (v3.1 and v3.0) so neither overview page lingers in the sidebar.
|
||||
if (HIDDEN_TAGS.has(tagSlug)) {
|
||||
for (const baseDir of [
|
||||
join(process.cwd(), 'content/reference/api-reference'),
|
||||
join(process.cwd(), 'content/reference/v3/api-reference'),
|
||||
]) {
|
||||
const hidden = join(baseDir, tagSlug, 'index.mdx');
|
||||
if (existsSync(hidden)) {
|
||||
rmSync(hidden);
|
||||
console.log(`Removed hidden tag: ${hidden}`);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
// Tag declared in spec.tags but no operations reference it — clean up any stale index.mdx from a prior run.
|
||||
if (ops31.length === 0 && ops3.length === 0) {
|
||||
for (const baseDir of [
|
||||
join(process.cwd(), 'content/reference/api-reference'),
|
||||
join(process.cwd(), 'content/reference/v3/api-reference'),
|
||||
]) {
|
||||
const stale = join(baseDir, tagSlug, 'index.mdx');
|
||||
if (existsSync(stale)) {
|
||||
rmSync(stale);
|
||||
console.log(`Removed stale: ${stale}`);
|
||||
}
|
||||
}
|
||||
continue;
|
||||
}
|
||||
|
||||
const tagDescription = tagDescriptions[tagName] || `${tagName} API endpoints`;
|
||||
// Display-title overrides for tags whose OpenAPI name is stale (e.g. the
|
||||
// tool router is now Sessions). Keyed by slug.
|
||||
const displayTitle = TITLE_OVERRIDES[tagSlug] ?? tagName;
|
||||
const overview = readOverview(tagSlug);
|
||||
// Body above the endpoints table: hand-written overview when present,
|
||||
// otherwise the thin OpenAPI tag description.
|
||||
const body = overview ?? tagDescription;
|
||||
const genComment = overview
|
||||
? `{/* Auto-generated from OpenAPI spec. Edit the overview at api-overviews/${tagSlug}.mdx, not this file. */}`
|
||||
: '{/* Auto-generated from OpenAPI spec. Do not edit directly. */}';
|
||||
|
||||
// Only generate v3.1 index page if the tag has v3.1 operations
|
||||
if (ops31.length > 0) {
|
||||
const v3ByOpId: Record<string, OperationEntry> = {};
|
||||
for (const op of ops3) {
|
||||
v3ByOpId[op.operationId] = op;
|
||||
}
|
||||
|
||||
const endpoints = ops31.map(op => {
|
||||
const v3Op = v3ByOpId[op.operationId];
|
||||
return {
|
||||
method: op.method,
|
||||
pathV31: op.path,
|
||||
pathV3: v3Op ? v3Op.path : op.path.replace('/v3.1/', '/v3/'),
|
||||
summary: op.summary,
|
||||
href: `/reference/api-reference/${tagSlug}/${op.operationId}`,
|
||||
};
|
||||
});
|
||||
|
||||
const content = `---
|
||||
title: ${displayTitle}
|
||||
description: "${tagDescription}"
|
||||
---
|
||||
|
||||
${genComment}
|
||||
|
||||
${body}
|
||||
|
||||
## Endpoints
|
||||
|
||||
<ApiEndpointsTable endpoints={${JSON.stringify(endpoints)}} />
|
||||
`;
|
||||
|
||||
const folderPath = join(outputDir, tagSlug);
|
||||
mkdirSync(folderPath, { recursive: true });
|
||||
writeFileSync(join(folderPath, 'index.mdx'), content);
|
||||
console.log(`Generated: ${tagSlug}/index.mdx`);
|
||||
}
|
||||
|
||||
// Also generate v3 index page with v3-specific hrefs
|
||||
if (ops3.length > 0) {
|
||||
const v3Endpoints = ops3.map(op => ({
|
||||
method: op.method,
|
||||
pathV31: op.path.replace('/v3/', '/v3.1/'),
|
||||
pathV3: op.path,
|
||||
summary: op.summary,
|
||||
href: `/reference/v3/api-reference/${tagSlug}/${op.operationId}`,
|
||||
}));
|
||||
|
||||
const v3Content = `---
|
||||
title: ${displayTitle}
|
||||
description: "${tagDescription}"
|
||||
---
|
||||
|
||||
${genComment}
|
||||
|
||||
${body}
|
||||
|
||||
## Endpoints
|
||||
|
||||
<ApiEndpointsTable endpoints={${JSON.stringify(v3Endpoints)}} />
|
||||
`;
|
||||
|
||||
const v3FolderPath = join(process.cwd(), 'content/reference/v3/api-reference', tagSlug);
|
||||
mkdirSync(v3FolderPath, { recursive: true });
|
||||
writeFileSync(join(v3FolderPath, 'index.mdx'), v3Content);
|
||||
console.log(`Generated: v3/api-reference/${tagSlug}/index.mdx`);
|
||||
}
|
||||
}
|
||||
|
||||
console.log('Done generating API index pages');
|
||||
}
|
||||
|
||||
generateIndexPages();
|
||||
@@ -0,0 +1,252 @@
|
||||
/**
|
||||
* Meta Tools Generator Script
|
||||
*
|
||||
* Fetches meta tool definitions from the Composio Tool Router API and generates:
|
||||
* - /public/data/meta-tools.json (complete meta tool schemas from API)
|
||||
* - /content/toolkits/meta-tools/meta.json (sidebar navigation)
|
||||
* - /content/toolkits/meta-tools/index.mdx (overview page)
|
||||
* - /content/toolkits/meta-tools/{name}.mdx (individual tool pages)
|
||||
*
|
||||
* All output is derived from the API response — no hardcoded tool definitions.
|
||||
*
|
||||
* Run: bun run generate:meta-tools
|
||||
*
|
||||
* Environment variables:
|
||||
* - COMPOSIO_API_KEY (required)
|
||||
* - COMPOSIO_API_BASE (optional, defaults to https://backend.composio.dev/api/v3)
|
||||
*/
|
||||
|
||||
import { writeFile, mkdir, readdir, unlink } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { fetchWithRetry } from './fetch-with-retry';
|
||||
import { META_TOOL_OVERRIDES } from '../lib/meta-tool-overrides';
|
||||
|
||||
const API_BASE = process.env.COMPOSIO_API_BASE || 'https://backend.composio.dev/api/v3';
|
||||
const API_KEY = process.env.COMPOSIO_API_KEY;
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error('Error: COMPOSIO_API_KEY environment variable is required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const DATA_DIR = join(process.cwd(), 'public/data');
|
||||
const CONTENT_DIR = join(process.cwd(), 'content/toolkits/meta-tools');
|
||||
|
||||
async function createSession(): Promise<string> {
|
||||
console.log('Creating session...');
|
||||
|
||||
const response = await fetchWithRetry(`${API_BASE}/tool_router/session`, {
|
||||
method: 'POST',
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': API_KEY!,
|
||||
},
|
||||
body: JSON.stringify({
|
||||
user_id: 'default',
|
||||
config: {
|
||||
autoManageConnections: true,
|
||||
recipesEnabled: true,
|
||||
enableWaitForConnections: true,
|
||||
},
|
||||
}),
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text();
|
||||
throw new Error(`Failed to create session: ${response.status} ${response.statusText}\n${body}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const sessionId = data.session_id;
|
||||
|
||||
if (!sessionId) {
|
||||
throw new Error('No session_id in response');
|
||||
}
|
||||
|
||||
console.log(` Session created: ${sessionId}`);
|
||||
console.log(` Tools available: ${(data.tool_router_tools || []).join(', ')}`);
|
||||
return sessionId;
|
||||
}
|
||||
|
||||
async function fetchMetaTools(sessionId: string): Promise<any[]> {
|
||||
console.log('Fetching meta tools with schemas...');
|
||||
|
||||
const response = await fetchWithRetry(`${API_BASE}/tool_router/session/${sessionId}/tools`, {
|
||||
headers: {
|
||||
'x-api-key': API_KEY!,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
const body = await response.text();
|
||||
throw new Error(`Failed to fetch meta tools: ${response.status} ${response.statusText}\n${body}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const tools = data.items || data;
|
||||
|
||||
if (!Array.isArray(tools)) {
|
||||
throw new Error('Expected array of tools in response');
|
||||
}
|
||||
|
||||
console.log(` Found ${tools.length} meta tools`);
|
||||
return tools;
|
||||
}
|
||||
|
||||
function transformTool(raw: any) {
|
||||
const slug: string = raw.slug || '';
|
||||
|
||||
return {
|
||||
slug,
|
||||
name: raw.name || '',
|
||||
displayName: raw.name?.replace(/_/g, ' ').replace(/\b\w/g, (c: string) => c.toUpperCase()) || slug,
|
||||
description: raw.description || '',
|
||||
tags: raw.tags || [],
|
||||
toolkit: raw.toolkit || null,
|
||||
inputParameters: raw.input_parameters || {},
|
||||
responseSchema: raw.output_parameters || {},
|
||||
};
|
||||
}
|
||||
|
||||
/** Derive a short page slug from the tool slug: COMPOSIO_SEARCH_TOOLS -> search_tools */
|
||||
function pageSlug(toolSlug: string): string {
|
||||
return toolSlug.toLowerCase().replace('composio_', '');
|
||||
}
|
||||
|
||||
/** Truncate description to first sentence for index table */
|
||||
function briefDescription(description: string): string {
|
||||
// Strip markdown and leading whitespace
|
||||
const cleaned = description.replace(/\*\*/g, '').replace(/__/g, '').replace(/\n+/g, ' ').trim();
|
||||
const firstSentence = cleaned.split(/\.(\s|$)/)[0];
|
||||
if (firstSentence.length > 120) {
|
||||
return firstSentence.slice(0, 117) + '...';
|
||||
}
|
||||
return firstSentence;
|
||||
}
|
||||
|
||||
/** One-line summary for the index table. Prefer hand-written override copy, fall back to the API description. */
|
||||
function indexLine(tool: any): string {
|
||||
const override = META_TOOL_OVERRIDES[tool.slug];
|
||||
if (override) {
|
||||
// First sentence of the hand-written summary keeps the table tight.
|
||||
const firstSentence = override.summary.split(/\.(\s|$)/)[0].trim();
|
||||
return firstSentence.replace(/\|/g, '\\|');
|
||||
}
|
||||
return briefDescription(tool.description).replace(/\|/g, '\\|');
|
||||
}
|
||||
|
||||
/** Generate the index.mdx overview page — Modal-voice intro plus a one-line-per-tool table */
|
||||
function generateIndexMdx(tools: any[]): string {
|
||||
let content = `---
|
||||
title: Meta Tools
|
||||
description: The system tools every Composio session gives your agent to discover, authenticate, execute, and process tools at runtime.
|
||||
keywords: [meta tools, session]
|
||||
---
|
||||
|
||||
{/* Auto-generated by scripts/generate-meta-tools.ts — do not edit manually. To change the intro prose or a tool's one-line summary, edit the template and the override map in lib/meta-tool-overrides.ts. */}
|
||||
|
||||
import { Callout } from 'fumadocs-ui/components/callout';
|
||||
|
||||
Every Composio [session](/docs/how-composio-works) hands your agent a small set of meta tools instead of hundreds of raw tool definitions. The agent uses them to find the right tools for a task, connect the accounts those tools need, execute them, and process the results, all at runtime and all sharing one \`session_id\`.
|
||||
|
||||
This keeps your context window small: you load a handful of meta tools, not a catalog of 500+ apps. The agent searches for what it needs when it needs it.
|
||||
|
||||
A typical workflow runs in order: call \`COMPOSIO_SEARCH_TOOLS\` to discover tools and open a session, call \`COMPOSIO_MANAGE_CONNECTIONS\` if a toolkit is not yet connected, then run the tools with \`COMPOSIO_MULTI_EXECUTE_TOOL\`. Reach for the workbench and bash tools when responses are large enough to process out of context.
|
||||
|
||||
| Tool | What it does |
|
||||
|------|--------------|
|
||||
`;
|
||||
|
||||
for (const tool of tools) {
|
||||
content += `| [\`${tool.slug}\`](/toolkits/meta-tools/${pageSlug(tool.slug)}) | ${indexLine(tool)} |\n`;
|
||||
}
|
||||
|
||||
content += `
|
||||
<Callout type="warn">
|
||||
These schemas are for reference only. We do not guarantee backward compatibility for parameter names or response shapes, so do not rely on them as structured type definitions in your code.
|
||||
</Callout>
|
||||
`;
|
||||
|
||||
return content;
|
||||
}
|
||||
|
||||
/** Generate an individual tool MDX page */
|
||||
function generateToolMdx(tool: any): string {
|
||||
const desc = briefDescription(tool.description).replace(/"/g, '\\"');
|
||||
|
||||
return `---
|
||||
title: ${tool.displayName}
|
||||
description: "${desc}"
|
||||
keywords: [${tool.slug}, meta tool]
|
||||
---
|
||||
|
||||
{/* Auto-generated by scripts/generate-meta-tools.ts — do not edit manually */}
|
||||
|
||||
import { MetaToolDetailServer } from '@/components/meta-tools/meta-tool-page';
|
||||
|
||||
<MetaToolDetailServer slug="${tool.slug}" />
|
||||
`;
|
||||
}
|
||||
|
||||
/** Generate meta.json for sidebar navigation — index.mdx is the folder page */
|
||||
function generateMetaJson(tools: any[]): string {
|
||||
const pages = tools.map(t => pageSlug(t.slug));
|
||||
return JSON.stringify({ title: 'Meta Tools', defaultOpen: true, pages }, null, 2) + '\n';
|
||||
}
|
||||
|
||||
async function cleanGeneratedMdx() {
|
||||
try {
|
||||
const files = await readdir(CONTENT_DIR);
|
||||
for (const file of files) {
|
||||
if (file.endsWith('.mdx') || file === 'meta.json') {
|
||||
await unlink(join(CONTENT_DIR, file));
|
||||
}
|
||||
}
|
||||
} catch {
|
||||
// Directory doesn't exist yet, that's fine
|
||||
}
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Starting meta tools generation...\n');
|
||||
|
||||
await mkdir(DATA_DIR, { recursive: true });
|
||||
await mkdir(CONTENT_DIR, { recursive: true });
|
||||
|
||||
const sessionId = await createSession();
|
||||
const rawTools = await fetchMetaTools(sessionId);
|
||||
const metaTools = rawTools.map(transformTool);
|
||||
|
||||
// Sort alphabetically by slug
|
||||
metaTools.sort((a, b) => a.slug.localeCompare(b.slug));
|
||||
|
||||
// 1. Write JSON data
|
||||
await writeFile(join(DATA_DIR, 'meta-tools.json'), JSON.stringify(metaTools, null, 2));
|
||||
console.log(`\nWrote public/data/meta-tools.json (~${Math.round(JSON.stringify(metaTools).length / 1024)}KB)`);
|
||||
|
||||
// 2. Clean old generated MDX files and regenerate
|
||||
await cleanGeneratedMdx();
|
||||
|
||||
// 3. Write meta.json
|
||||
await writeFile(join(CONTENT_DIR, 'meta.json'), generateMetaJson(metaTools));
|
||||
console.log('Wrote content/toolkits/meta-tools/meta.json');
|
||||
|
||||
// 4. Write index.mdx
|
||||
await writeFile(join(CONTENT_DIR, 'index.mdx'), generateIndexMdx(metaTools));
|
||||
console.log('Wrote content/toolkits/meta-tools/index.mdx');
|
||||
|
||||
// 5. Write individual tool pages
|
||||
for (const tool of metaTools) {
|
||||
const filename = `${pageSlug(tool.slug)}.mdx`;
|
||||
await writeFile(join(CONTENT_DIR, filename), generateToolMdx(tool));
|
||||
console.log(`Wrote content/toolkits/meta-tools/${filename}`);
|
||||
}
|
||||
|
||||
console.log(`\nGeneration complete! ${metaTools.length} meta tools.`);
|
||||
console.log(`Tools: ${metaTools.map(t => t.slug).join(', ')}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,362 @@
|
||||
/**
|
||||
* Toolkit Generator Script
|
||||
*
|
||||
* Fetches all toolkits from Composio API and generates:
|
||||
* - /public/data/toolkits.json (full data with tools & triggers - for detail pages)
|
||||
* - /public/data/toolkits-list.json (light version without tools/triggers - for landing page)
|
||||
*
|
||||
* Run: bun run generate:toolkits
|
||||
*/
|
||||
|
||||
import { mkdir, writeFile } from 'fs/promises';
|
||||
import { join } from 'path';
|
||||
import { fetchWithRetry } from './fetch-with-retry';
|
||||
|
||||
const API_BASE = process.env.COMPOSIO_API_BASE || 'https://backend.composio.dev/api/v3';
|
||||
const API_KEY = process.env.COMPOSIO_API_KEY;
|
||||
|
||||
if (!API_KEY) {
|
||||
console.error('Error: COMPOSIO_API_KEY environment variable is required');
|
||||
process.exit(1);
|
||||
}
|
||||
|
||||
const OUTPUT_DIR = join(process.cwd(), 'public/data');
|
||||
|
||||
interface Tool {
|
||||
slug: string;
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface Trigger {
|
||||
slug: string;
|
||||
name: string;
|
||||
description: string;
|
||||
}
|
||||
|
||||
interface AuthConfigField {
|
||||
name: string;
|
||||
displayName: string;
|
||||
type: string;
|
||||
description: string;
|
||||
required: boolean;
|
||||
default?: string | null;
|
||||
}
|
||||
|
||||
interface AuthConfigDetail {
|
||||
mode: string;
|
||||
name: string;
|
||||
fields: {
|
||||
auth_config_creation: {
|
||||
required: AuthConfigField[];
|
||||
optional: AuthConfigField[];
|
||||
};
|
||||
connected_account_initiation: {
|
||||
required: AuthConfigField[];
|
||||
optional: AuthConfigField[];
|
||||
};
|
||||
};
|
||||
}
|
||||
|
||||
interface Toolkit {
|
||||
slug: string;
|
||||
name: string;
|
||||
logo: string | null;
|
||||
description: string;
|
||||
category: string | null;
|
||||
authSchemes: string[];
|
||||
composioManagedAuthSchemes?: string[];
|
||||
toolCount: number;
|
||||
triggerCount: number;
|
||||
version: string | null;
|
||||
tools: Tool[];
|
||||
triggers: Trigger[];
|
||||
authConfigDetails?: AuthConfigDetail[];
|
||||
}
|
||||
|
||||
// The backend silently caps `limit` at 1000 per page and defaults to usage
|
||||
// ordering, so a single request returns only the top-1000 toolkits — about half
|
||||
// the catalog. Request the cap and follow `next_cursor` until exhausted.
|
||||
// MAX_PAGES is a runaway guard well above the real catalog (~2.1k → 3 pages).
|
||||
const TOOLKITS_PAGE_LIMIT = 1000;
|
||||
const TOOLKITS_MAX_PAGES = 12;
|
||||
|
||||
async function fetchToolkits(): Promise<any[]> {
|
||||
console.log('Fetching toolkits from API...');
|
||||
|
||||
const items: any[] = [];
|
||||
// Pages can overlap when the catalog shifts between cursor fetches; keep the
|
||||
// first occurrence so API-provided ordering stays stable.
|
||||
const seen = new Set<string>();
|
||||
let cursor: string | undefined;
|
||||
|
||||
for (let page = 0; page < TOOLKITS_MAX_PAGES; page++) {
|
||||
const params = new URLSearchParams({ limit: String(TOOLKITS_PAGE_LIMIT) });
|
||||
if (cursor) params.set('cursor', cursor);
|
||||
|
||||
const response = await fetchWithRetry(`${API_BASE}/toolkits?${params}`, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': API_KEY!,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
throw new Error(`Failed to fetch toolkits: ${response.status} ${response.statusText}`);
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const pageItems: any[] = data.items || data;
|
||||
for (const item of pageItems) {
|
||||
const slug = item?.slug?.toLowerCase();
|
||||
if (slug) {
|
||||
if (seen.has(slug)) continue;
|
||||
seen.add(slug);
|
||||
}
|
||||
items.push(item);
|
||||
}
|
||||
|
||||
cursor = data.next_cursor ?? undefined;
|
||||
if (!cursor) return items;
|
||||
}
|
||||
|
||||
// Failing beats silently publishing a truncated catalog — that is the exact
|
||||
// bug this pagination loop exists to prevent.
|
||||
throw new Error(
|
||||
`Toolkit catalog exceeds ${TOOLKITS_MAX_PAGES} pages of ${TOOLKITS_PAGE_LIMIT}; raise TOOLKITS_MAX_PAGES`
|
||||
);
|
||||
}
|
||||
|
||||
async function fetchToolkitChangelog(): Promise<Map<string, string>> {
|
||||
console.log('Fetching toolkit changelog...');
|
||||
|
||||
const response = await fetchWithRetry(`${API_BASE}/toolkits/changelog`, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': API_KEY!,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) {
|
||||
console.warn(`Failed to fetch changelog: ${response.status}`);
|
||||
return new Map();
|
||||
}
|
||||
|
||||
const data = await response.json();
|
||||
const versionMap = new Map<string, string>();
|
||||
|
||||
// Response format: { items: [{ slug, name, display_name, versions: [{ version, changelog }] }] }
|
||||
const items = data.items || [];
|
||||
for (const entry of items) {
|
||||
const slug = entry.slug?.toLowerCase();
|
||||
const latestVersion = entry.versions?.[0]?.version;
|
||||
if (slug && latestVersion) {
|
||||
versionMap.set(slug, latestVersion);
|
||||
}
|
||||
}
|
||||
|
||||
console.log(`Found versions for ${versionMap.size} toolkits`);
|
||||
return versionMap;
|
||||
}
|
||||
|
||||
async function fetchToolsForToolkit(slug: string): Promise<Tool[]> {
|
||||
const response = await fetchWithRetry(`${API_BASE}/tools?toolkit_slug=${slug}&toolkit_versions=latest&limit=1000`, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': API_KEY!,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) return [];
|
||||
|
||||
const data = await response.json();
|
||||
const rawItems = data.items || data;
|
||||
const items = Array.isArray(rawItems) ? rawItems : [];
|
||||
|
||||
return items.filter((raw: any) => raw && typeof raw === 'object').map((raw: any) => ({
|
||||
slug: raw.slug || '',
|
||||
name: raw.name || raw.display_name || raw.slug || '',
|
||||
description: raw.description || '',
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchTriggersForToolkit(slug: string): Promise<Trigger[]> {
|
||||
const response = await fetchWithRetry(`${API_BASE}/triggers_types?toolkit_slugs=${slug}&toolkit_versions=latest`, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': API_KEY!,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) return [];
|
||||
|
||||
const data = await response.json();
|
||||
const rawItems = data.items || data;
|
||||
const items = Array.isArray(rawItems) ? rawItems : [];
|
||||
|
||||
return items.filter((raw: any) => raw && typeof raw === 'object').map((raw: any) => ({
|
||||
slug: raw.slug || '',
|
||||
name: raw.name || raw.display_name || raw.slug || '',
|
||||
description: raw.description || '',
|
||||
}));
|
||||
}
|
||||
|
||||
async function fetchAuthConfigDetails(slug: string): Promise<AuthConfigDetail[]> {
|
||||
const response = await fetchWithRetry(`${API_BASE}/toolkits/${slug}`, {
|
||||
headers: {
|
||||
'Content-Type': 'application/json',
|
||||
'x-api-key': API_KEY!,
|
||||
},
|
||||
});
|
||||
|
||||
if (!response.ok) return [];
|
||||
|
||||
const data = await response.json();
|
||||
const authConfigDetails = data.auth_config_details || [];
|
||||
|
||||
return authConfigDetails.map((raw: any) => ({
|
||||
mode: raw.mode || '',
|
||||
name: raw.name || raw.mode || '',
|
||||
fields: {
|
||||
auth_config_creation: {
|
||||
required: (raw.fields?.auth_config_creation?.required || []).map((f: any) => ({
|
||||
name: f.name || '',
|
||||
displayName: f.displayName || f.name || '',
|
||||
type: f.type || 'string',
|
||||
description: f.description || '',
|
||||
required: f.required ?? true,
|
||||
default: f.default ?? null,
|
||||
})),
|
||||
optional: (raw.fields?.auth_config_creation?.optional || []).map((f: any) => ({
|
||||
name: f.name || '',
|
||||
displayName: f.displayName || f.name || '',
|
||||
type: f.type || 'string',
|
||||
description: f.description || '',
|
||||
required: f.required ?? false,
|
||||
default: f.default ?? null,
|
||||
})),
|
||||
},
|
||||
connected_account_initiation: {
|
||||
required: (raw.fields?.connected_account_initiation?.required || []).map((f: any) => ({
|
||||
name: f.name || '',
|
||||
displayName: f.displayName || f.name || '',
|
||||
type: f.type || 'string',
|
||||
description: f.description || '',
|
||||
required: f.required ?? true,
|
||||
default: f.default ?? null,
|
||||
})),
|
||||
optional: (raw.fields?.connected_account_initiation?.optional || []).map((f: any) => ({
|
||||
name: f.name || '',
|
||||
displayName: f.displayName || f.name || '',
|
||||
type: f.type || 'string',
|
||||
description: f.description || '',
|
||||
required: f.required ?? false,
|
||||
default: f.default ?? null,
|
||||
})),
|
||||
},
|
||||
},
|
||||
}));
|
||||
}
|
||||
|
||||
function transformToolkit(raw: any): Toolkit {
|
||||
const authSchemes = raw.auth_schemes || raw.authSchemes || [];
|
||||
const composioManaged = raw.composio_managed_auth_schemes || raw.composioManagedAuthSchemes || [];
|
||||
|
||||
return {
|
||||
slug: raw.slug?.toLowerCase() || '',
|
||||
name: raw.name || raw.slug || '',
|
||||
logo: raw.meta?.logo || raw.logo || null,
|
||||
description: raw.meta?.description || raw.description || '',
|
||||
category: raw.meta?.categories?.[0]?.name || raw.meta?.categories?.[0] || null,
|
||||
authSchemes,
|
||||
...(composioManaged.length > 0 ? { composioManagedAuthSchemes: composioManaged } : {}),
|
||||
toolCount: raw.tool_count || raw.toolCount || 0,
|
||||
triggerCount: raw.trigger_count || raw.triggerCount || 0,
|
||||
version: null,
|
||||
tools: [],
|
||||
triggers: [],
|
||||
};
|
||||
}
|
||||
|
||||
async function main() {
|
||||
console.log('Starting toolkit generation...\n');
|
||||
|
||||
// Create output directory
|
||||
await mkdir(OUTPUT_DIR, { recursive: true });
|
||||
|
||||
// Fetch all toolkits and changelog in parallel
|
||||
const [rawToolkits, versionMap] = await Promise.all([
|
||||
fetchToolkits(),
|
||||
fetchToolkitChangelog(),
|
||||
]);
|
||||
console.log(`Found ${rawToolkits.length} toolkits\n`);
|
||||
|
||||
// Transform toolkits
|
||||
const toolkits: Toolkit[] = rawToolkits.map(transformToolkit);
|
||||
|
||||
// Add versions from changelog
|
||||
for (const toolkit of toolkits) {
|
||||
toolkit.version = versionMap.get(toolkit.slug) || null;
|
||||
}
|
||||
|
||||
// Fetch tools, triggers, and auth config details for each toolkit in batches.
|
||||
// Each toolkit fires 3 requests in parallel, so batchSize N = ~3N concurrent
|
||||
// requests. Kept low to soften burst pressure on the staging rate limit
|
||||
// (2000 req/min); fetchWithRetry handles the remaining 429s with backoff.
|
||||
console.log('Fetching tools, triggers, and auth config details...');
|
||||
const batchSize = 5;
|
||||
let completed = 0;
|
||||
|
||||
for (let i = 0; i < toolkits.length; i += batchSize) {
|
||||
const batch = toolkits.slice(i, i + batchSize);
|
||||
|
||||
await Promise.all(
|
||||
batch.map(async (toolkit) => {
|
||||
const [tools, triggers, authConfigDetails] = await Promise.all([
|
||||
fetchToolsForToolkit(toolkit.slug.toUpperCase()),
|
||||
fetchTriggersForToolkit(toolkit.slug.toUpperCase()),
|
||||
fetchAuthConfigDetails(toolkit.slug),
|
||||
]);
|
||||
|
||||
toolkit.tools = tools;
|
||||
toolkit.triggers = triggers;
|
||||
toolkit.toolCount = tools.length;
|
||||
toolkit.triggerCount = triggers.length;
|
||||
toolkit.authConfigDetails = authConfigDetails.length > 0 ? authConfigDetails : undefined;
|
||||
|
||||
completed++;
|
||||
process.stdout.write(`\r Progress: ${completed}/${toolkits.length}`);
|
||||
})
|
||||
);
|
||||
}
|
||||
|
||||
console.log('\n');
|
||||
|
||||
// Write full file (for detail pages - read from filesystem)
|
||||
await writeFile(
|
||||
join(OUTPUT_DIR, 'toolkits.json'),
|
||||
JSON.stringify(toolkits, null, 2)
|
||||
);
|
||||
|
||||
// Write light file (for landing page - imported in client component)
|
||||
// Excludes tools and triggers arrays to keep bundle size small
|
||||
const toolkitsLight = toolkits.map(({ slug, name, logo, category, toolCount, triggerCount }) => ({
|
||||
slug, name, logo, category, toolCount, triggerCount,
|
||||
}));
|
||||
await writeFile(
|
||||
join(OUTPUT_DIR, 'toolkits-list.json'),
|
||||
JSON.stringify(toolkitsLight, null, 2)
|
||||
);
|
||||
|
||||
const fullSizeKB = Math.round(JSON.stringify(toolkits).length / 1024);
|
||||
const lightSizeKB = Math.round(JSON.stringify(toolkitsLight).length / 1024);
|
||||
console.log('Generation complete!');
|
||||
console.log(` Full: public/data/toolkits.json (~${fullSizeKB}KB)`);
|
||||
console.log(` Light: public/data/toolkits-list.json (~${lightSizeKB}KB)`);
|
||||
console.log(` Toolkits: ${toolkits.length}`);
|
||||
}
|
||||
|
||||
main().catch((error) => {
|
||||
console.error('Fatal error:', error);
|
||||
process.exit(1);
|
||||
});
|
||||
@@ -0,0 +1,3 @@
|
||||
import { createMdxPlugin } from 'fumadocs-mdx/bun';
|
||||
|
||||
Bun.plugin(createMdxPlugin());
|
||||
@@ -0,0 +1,102 @@
|
||||
import { algoliasearch } from 'algoliasearch';
|
||||
import {
|
||||
ALGOLIA_DEFAULT_APP_ID,
|
||||
ALGOLIA_DEFAULT_INDEX_NAME,
|
||||
getAlgoliaSearchDocuments,
|
||||
} from '@/lib/search-index';
|
||||
|
||||
function requireEnv(name: string, fallback?: string): string {
|
||||
const value = process.env[name] ?? fallback;
|
||||
if (!value) {
|
||||
throw new Error(`Missing required environment variable: ${name}`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
const dryRun = process.argv.includes('--dry-run');
|
||||
const showSamples = process.argv.includes('--samples');
|
||||
const indexName =
|
||||
process.env.ALGOLIA_INDEX_NAME ?? process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME ?? ALGOLIA_DEFAULT_INDEX_NAME;
|
||||
|
||||
const records = await getAlgoliaSearchDocuments();
|
||||
|
||||
if (records.length === 0) {
|
||||
throw new Error('Refusing to sync an empty Algolia docs index.');
|
||||
}
|
||||
|
||||
if (dryRun) {
|
||||
const uniquePages = new Set(records.map((record) => record.page_id)).size;
|
||||
console.log(`Prepared ${records.length} Algolia records across ${uniquePages} docs pages for index "${indexName}".`);
|
||||
|
||||
if (showSamples) {
|
||||
console.log(JSON.stringify(records.slice(0, 5), null, 2));
|
||||
}
|
||||
|
||||
process.exit(0);
|
||||
}
|
||||
|
||||
const appId = requireEnv(
|
||||
'ALGOLIA_APP_ID',
|
||||
process.env.NEXT_PUBLIC_ALGOLIA_APP_ID ?? ALGOLIA_DEFAULT_APP_ID,
|
||||
);
|
||||
const adminApiKey = requireEnv('ALGOLIA_ADMIN_API_KEY');
|
||||
const client = algoliasearch(appId, adminApiKey);
|
||||
|
||||
console.log(`Configuring Algolia index "${indexName}"...`);
|
||||
await client.setSettings({
|
||||
indexName,
|
||||
indexSettings: {
|
||||
attributeForDistinct: 'page_id',
|
||||
attributesToRetrieve: [
|
||||
'objectID',
|
||||
'title',
|
||||
'description',
|
||||
'section',
|
||||
'content',
|
||||
'url',
|
||||
'section_id',
|
||||
'breadcrumbs',
|
||||
'page_id',
|
||||
'type',
|
||||
],
|
||||
searchableAttributes: [
|
||||
'unordered(title)',
|
||||
'unordered(keywords)',
|
||||
'unordered(slug)',
|
||||
'unordered(section)',
|
||||
'unordered(headings)',
|
||||
'unordered(tool_names)',
|
||||
'unordered(tool_slugs)',
|
||||
'unordered(description)',
|
||||
'unordered(content)',
|
||||
],
|
||||
customRanking: [
|
||||
'desc(page_rank)',
|
||||
'desc(toolkit_popularity)',
|
||||
'desc(section_rank)',
|
||||
'asc(position)',
|
||||
],
|
||||
attributesForFaceting: [
|
||||
'filterOnly(type)',
|
||||
'filterOnly(lang)',
|
||||
'searchable(tags)',
|
||||
],
|
||||
attributesToHighlight: ['title', 'section', 'content'],
|
||||
attributesToSnippet: ['content:30'],
|
||||
highlightPreTag: '<mark>',
|
||||
highlightPostTag: '</mark>',
|
||||
ignorePlurals: true,
|
||||
minProximity: 1,
|
||||
removeStopWords: false,
|
||||
removeWordsIfNoResults: 'lastWords',
|
||||
typoTolerance: true,
|
||||
advancedSyntax: true,
|
||||
},
|
||||
});
|
||||
|
||||
console.log(`Replacing ${records.length} records in Algolia index "${indexName}"...`);
|
||||
await client.replaceAllObjects({
|
||||
indexName,
|
||||
objects: records.map((record) => ({ ...record })),
|
||||
});
|
||||
console.log(`Synced Algolia docs search index "${indexName}".`);
|
||||
@@ -0,0 +1,56 @@
|
||||
import { liteClient } from 'algoliasearch/lite';
|
||||
import type { AlgoliaDocsRecord } from '@/lib/search-index';
|
||||
import { ALGOLIA_DEFAULT_APP_ID, ALGOLIA_DEFAULT_INDEX_NAME } from '@/lib/search-index';
|
||||
|
||||
function requireEnv(name: string, fallback?: string): string {
|
||||
const value = process.env[name] ?? fallback;
|
||||
if (!value) throw new Error(`Missing required environment variable: ${name}`);
|
||||
return value;
|
||||
}
|
||||
|
||||
const appId = requireEnv('ALGOLIA_APP_ID', process.env.NEXT_PUBLIC_ALGOLIA_APP_ID ?? ALGOLIA_DEFAULT_APP_ID);
|
||||
const searchApiKey = requireEnv(
|
||||
'ALGOLIA_SEARCH_API_KEY',
|
||||
process.env.NEXT_PUBLIC_ALGOLIA_SEARCH_API_KEY ?? process.env.ALGOLIA_ADMIN_API_KEY,
|
||||
);
|
||||
const indexName = process.env.ALGOLIA_INDEX_NAME ?? process.env.NEXT_PUBLIC_ALGOLIA_INDEX_NAME ?? ALGOLIA_DEFAULT_INDEX_NAME;
|
||||
const queries = process.argv.slice(2);
|
||||
|
||||
if (queries.length === 0) {
|
||||
queries.push(
|
||||
'quickstart',
|
||||
'oauth auth config',
|
||||
'create connected account',
|
||||
'gmail send email',
|
||||
'tool router session search',
|
||||
'webhook verification',
|
||||
);
|
||||
}
|
||||
|
||||
const client = liteClient(appId, searchApiKey);
|
||||
|
||||
for (const query of queries) {
|
||||
const result = await client.searchForHits<AlgoliaDocsRecord>({
|
||||
requests: [
|
||||
{
|
||||
type: 'default',
|
||||
indexName,
|
||||
query,
|
||||
hitsPerPage: 5,
|
||||
distinct: true,
|
||||
getRankingInfo: true,
|
||||
clickAnalytics: true,
|
||||
},
|
||||
],
|
||||
});
|
||||
|
||||
const response = result.results[0];
|
||||
console.log(`\n## ${query}`);
|
||||
response.hits.forEach((hit, index) => {
|
||||
const section = hit.section ? ` — ${hit.section}` : '';
|
||||
const snippet = hit.content.replace(/\s+/g, ' ').slice(0, 180);
|
||||
console.log(`${index + 1}. ${hit.title}${section}`);
|
||||
console.log(` ${hit.url}`);
|
||||
console.log(` ${snippet}${snippet.length === 180 ? '…' : ''}`);
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,174 @@
|
||||
import { readFile } from 'node:fs/promises';
|
||||
import { glob } from 'node:fs/promises';
|
||||
import { resolve } from 'node:path';
|
||||
import {
|
||||
type FileObject,
|
||||
printErrors,
|
||||
scanURLs,
|
||||
validateFiles,
|
||||
} from 'next-validate-link';
|
||||
import GithubSlugger from 'github-slugger';
|
||||
import {
|
||||
source,
|
||||
getReferenceSource,
|
||||
examplesSource,
|
||||
toolkitsSource,
|
||||
} from '../lib/source';
|
||||
|
||||
type AnySource =
|
||||
| typeof source
|
||||
| Awaited<ReturnType<typeof getReferenceSource>>
|
||||
| typeof examplesSource
|
||||
| typeof toolkitsSource;
|
||||
|
||||
type PageOf = ReturnType<AnySource['getPages']>[number];
|
||||
|
||||
/**
|
||||
* Extract heading anchors from raw MDX/markdown content.
|
||||
* Falls back to this when data.toc is unavailable (outside Next.js runtime).
|
||||
* Uses github-slugger to match rehype-slug's algorithm (handles Unicode, duplicate suffixes).
|
||||
*/
|
||||
function extractHeadingsFromContent(content: string): string[] {
|
||||
const slugger = new GithubSlugger();
|
||||
const headings: string[] = [];
|
||||
let inCodeBlock = false;
|
||||
for (const line of content.split('\n')) {
|
||||
if (line.startsWith('```')) {
|
||||
inCodeBlock = !inCodeBlock;
|
||||
continue;
|
||||
}
|
||||
if (inCodeBlock) continue;
|
||||
const match = line.match(/^#{1,6}\s+(.+)$/);
|
||||
if (match) {
|
||||
headings.push(slugger.slug(match[1]));
|
||||
}
|
||||
}
|
||||
return headings;
|
||||
}
|
||||
|
||||
/**
|
||||
* Get headings for a page, trying data.toc first then falling back to raw content parsing.
|
||||
*/
|
||||
async function getHeadingsForPage(page: PageOf): Promise<string[]> {
|
||||
if (page.data.toc?.length) {
|
||||
return page.data.toc.map((item: { url: string }) => item.url.slice(1));
|
||||
}
|
||||
if ('getText' in page.data) {
|
||||
try {
|
||||
const content = await (page.data as { getText: (mode: string) => Promise<string> }).getText('raw');
|
||||
return extractHeadingsFromContent(content);
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
if (page.absolutePath) {
|
||||
try {
|
||||
const content = await readFile(page.absolutePath, 'utf-8');
|
||||
return extractHeadingsFromContent(content);
|
||||
} catch {
|
||||
// fall through
|
||||
}
|
||||
}
|
||||
return [];
|
||||
}
|
||||
|
||||
/**
|
||||
* Build populate entries for a source, resolving headings asynchronously.
|
||||
*/
|
||||
async function buildPopulateEntries(src: AnySource) {
|
||||
return Promise.all(
|
||||
src.getPages().map(async (page: PageOf) => ({
|
||||
value: { slug: page.slugs },
|
||||
hashes: await getHeadingsForPage(page),
|
||||
})),
|
||||
);
|
||||
}
|
||||
|
||||
async function getDynamicToolkitEntries() {
|
||||
const raw = await readFile('public/data/toolkits.json', 'utf-8');
|
||||
const toolkits: { slug: string }[] = JSON.parse(raw);
|
||||
return toolkits.map((t) => ({ value: { slug: [t.slug] }, hashes: [] as string[] }));
|
||||
}
|
||||
|
||||
async function checkLinks() {
|
||||
const referenceSource = await getReferenceSource();
|
||||
const [docsEntries, refEntries, exampleEntries, toolkitEntries, dynamicToolkitEntries] = await Promise.all([
|
||||
buildPopulateEntries(source),
|
||||
buildPopulateEntries(referenceSource),
|
||||
buildPopulateEntries(examplesSource),
|
||||
buildPopulateEntries(toolkitsSource),
|
||||
getDynamicToolkitEntries(),
|
||||
]);
|
||||
|
||||
const scanned = await scanURLs({
|
||||
preset: 'next',
|
||||
populate: {
|
||||
// Keys must include (home) route group to match app directory structure
|
||||
'(home)/docs/[[...slug]]': docsEntries,
|
||||
'(home)/reference/[[...slug]]': refEntries,
|
||||
'(home)/examples/[[...slug]]': exampleEntries,
|
||||
'(home)/toolkits/[[...slug]]': [...toolkitEntries, ...dynamicToolkitEntries],
|
||||
},
|
||||
});
|
||||
|
||||
const errors = await validateFiles(await getFiles(), {
|
||||
scanned,
|
||||
markdown: {
|
||||
components: {
|
||||
Card: { attributes: ['href'] },
|
||||
},
|
||||
},
|
||||
checkRelativePaths: 'as-url',
|
||||
});
|
||||
|
||||
// Filter out API route URLs (these are valid but not detected as pages)
|
||||
const ignoredUrls = ['/llms.txt', '/llms-full.txt'];
|
||||
const filteredErrors = errors
|
||||
.map((fileError) => ({
|
||||
...fileError,
|
||||
errors: fileError.errors.filter((e) => !ignoredUrls.includes(e.url)),
|
||||
detected: fileError.detected.filter((d) => !ignoredUrls.includes(d[0] as string)),
|
||||
}))
|
||||
.filter((fileError) => fileError.errors.length > 0);
|
||||
|
||||
printErrors(filteredErrors, true);
|
||||
if (filteredErrors.length > 0) {
|
||||
process.exit(1);
|
||||
}
|
||||
}
|
||||
|
||||
async function getFiles(): Promise<FileObject[]> {
|
||||
const referenceSource = await getReferenceSource();
|
||||
const sources = [source, referenceSource, examplesSource, toolkitsSource];
|
||||
const allFiles: FileObject[] = [];
|
||||
|
||||
for (const src of sources) {
|
||||
const pages = src.getPages();
|
||||
for (const page of pages) {
|
||||
if (!page.absolutePath) continue;
|
||||
if (!page.absolutePath.endsWith('.mdx') && !page.absolutePath.endsWith('.md')) continue;
|
||||
// Skip OpenAPI-generated pages (they don't have getText)
|
||||
if (!('getText' in page.data)) continue;
|
||||
|
||||
allFiles.push({
|
||||
path: page.absolutePath,
|
||||
content: await (page.data as { getText: (mode: string) => Promise<string> }).getText('raw'),
|
||||
url: page.url,
|
||||
data: page.data,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// Scan any .md files under content/ not already covered by Fumadocs sources
|
||||
const coveredPaths = new Set(allFiles.map((f) => resolve(f.path)));
|
||||
const extraMdFiles = await Array.fromAsync(glob('content/**/*.md'));
|
||||
for (const filePath of extraMdFiles) {
|
||||
if (coveredPaths.has(resolve(filePath))) continue;
|
||||
const content = await readFile(filePath, 'utf-8');
|
||||
allFiles.push({ path: filePath, content });
|
||||
}
|
||||
|
||||
return allFiles;
|
||||
}
|
||||
|
||||
void checkLinks();
|
||||
Reference in New Issue
Block a user