/** * Generate Search Index * * Scans reference MDX files, AG-UI content, and registry data to produce * a search index JSON for the shell's Cmd-K search modal. * * Usage: npx tsx showcase/scripts/generate-search-index.ts * * Output: showcase/shell/src/data/search-index.json */ import fs from "fs"; import path from "path"; import { fileURLToPath } from "url"; const __dirname = path.dirname(fileURLToPath(import.meta.url)); const ROOT = path.resolve(__dirname, ".."); // MDX docs content now lives in shell-docs. shell-docs consumes the index // at build time for ; shell keeps a copy so its header search // (which links to docs routes) stays functional — destinations 301 across // to docs.showcase.copilotkit.ai. We SCAN from shell-docs (source of truth) // and WRITE to both. const SHELL_DOCS_DIR = path.join(ROOT, "shell-docs", "src"); const SHELL_DIR = path.join(ROOT, "shell", "src"); const CONTENT_ROOT = SHELL_DOCS_DIR; const OUTPUT_PATHS = [ path.join(SHELL_DOCS_DIR, "data", "search-index.json"), path.join(SHELL_DIR, "data", "search-index.json"), ]; interface SearchEntry { type: "page" | "reference" | "ag-ui"; title: string; subtitle: string; section: string; href: string; } const FRONTEND_SEARCH_PAGES = [ { id: "vue", name: "Vue", guidanceTitle: "Docs status" }, { id: "react-native", name: "React Native", guidanceTitle: "Docs status" }, { id: "angular", name: "Angular", guidanceTitle: "Docs status" }, { id: "slack", name: "Slack", guidanceTitle: "About early access" }, { id: "teams", name: "Teams", guidanceTitle: "About early access" }, ] as const; const FRONTEND_NAMES = new Map( FRONTEND_SEARCH_PAGES.map((frontend) => [frontend.id, frontend.name]), ); // Derive a human-readable section breadcrumb from a relative path. // e.g. "concepts/middleware" → "Concepts" // "sdk/js/client/middleware" → "JS SDK › @ag-ui/client" // "backend/copilot-runtime" → "Backend" const SECTION_LABELS: Record = { concepts: "Concepts", quickstart: "Quickstart", drafts: "Draft Proposals", tutorials: "Tutorials", development: "Development", "sdk/js": "JS SDK", "sdk/js/core": "JS SDK › @ag-ui/core", "sdk/js/client": "JS SDK › @ag-ui/client", "sdk/python": "Python SDK", "sdk/python/core": "Python SDK › ag_ui.core", "sdk/python/encoder": "Python SDK › ag_ui.encoder", }; function deriveSectionLabel(hrefPrefix: string, href: string): string { // Strip prefix to get relative path, then drop the filename const rel = href.slice(hrefPrefix.length + 1); // e.g. "concepts/middleware" const parts = rel.split("/"); if (parts.length <= 1) return ""; // top-level page, no section // Try longest prefix match first for (let len = parts.length - 1; len >= 1; len--) { const candidate = parts.slice(0, len).join("/"); if (SECTION_LABELS[candidate]) return SECTION_LABELS[candidate]; } // Fallback: capitalize first directory return ( parts[0].charAt(0).toUpperCase() + parts[0].slice(1).replace(/-/g, " ") ); } function extractTitle(content: string, filename: string): string { // Try frontmatter title const fmMatch = content.match(/^---[\s\S]*?title:\s*["']?(.+?)["']?\s*$/m); if (fmMatch) return fmMatch[1]; // Try first heading const h1Match = content.match(/^#\s+(.+)$/m); if (h1Match) return h1Match[1]; // Fall back to filename return filename.replace(".mdx", "").replace(/-/g, " "); } function extractDescription(content: string): string { // Strip frontmatter const stripped = content.replace(/^---[\s\S]*?---\n?/, ""); // Strip headings and find first paragraph const lines = stripped .split("\n") .filter( (l) => l.trim() && !l.startsWith("#") && !l.startsWith("import") && !l.startsWith("<") && !l.startsWith("```"), ); const first = lines[0]?.trim() || ""; return first.slice(0, 120); } function scanMdxDir( dir: string, hrefPrefix: string, type: "page" | "reference" | "ag-ui", allowList?: Set, ): SearchEntry[] { const entries: SearchEntry[] = []; function walk(currentDir: string, pathPrefix: string) { if (!fs.existsSync(currentDir)) return; const items = fs.readdirSync(currentDir, { withFileTypes: true }); for (const item of items) { if (item.isDirectory()) { walk(path.join(currentDir, item.name), `${pathPrefix}/${item.name}`); } else if (item.name.endsWith(".mdx")) { const slug = item.name.replace(".mdx", ""); const href = slug === "index" ? hrefPrefix + pathPrefix : `${hrefPrefix}${pathPrefix}/${slug}`; // When an allow list is provided, only include matching slugs. // The slug is the href with the prefix stripped and leading slash removed. if (allowList) { const relSlug = href.slice(hrefPrefix.length + 1); if (!allowList.has(relSlug)) continue; } const content = fs.readFileSync( path.join(currentDir, item.name), "utf-8", ); const title = extractTitle(content, item.name); const subtitle = extractDescription(content); const section = deriveSectionLabel(hrefPrefix, href); entries.push({ type, title, subtitle, section, href }); } } } walk(dir, ""); return entries; } function normalizeDocsSearchEntry(entry: SearchEntry): SearchEntry[] { const frontendPrefix = "/docs/frontends/"; if (!entry.href.startsWith(frontendPrefix)) return [entry]; const slugPath = entry.href.slice(frontendPrefix.length); if (slugPath === "using-these-docs") { return FRONTEND_SEARCH_PAGES.filter( (frontend) => frontend.guidanceTitle === "About early access", ).map((frontend) => ({ ...entry, title: `${frontend.name}: ${frontend.guidanceTitle}`, section: "Frontends", href: `/${frontend.id}/using-these-docs`, })); } if (slugPath === "docs-status") { return FRONTEND_SEARCH_PAGES.filter( (frontend) => frontend.guidanceTitle === "Docs status", ).map((frontend) => ({ ...entry, title: `${frontend.name}: ${frontend.guidanceTitle}`, section: "Frontends", href: `/${frontend.id}/using-these-docs`, })); } const [frontend, ...tail] = slugPath.split("/").filter(Boolean); if (!frontend || !FRONTEND_NAMES.has(frontend)) return []; return [ { ...entry, section: "Frontends", href: tail.length > 0 ? `/${frontend}/${tail.join("/")}` : `/${frontend}`, }, ]; } function main() { const entries: SearchEntry[] = []; // Static pages entries.push( { type: "page", title: "Home", subtitle: "Front door", section: "", href: "/", }, { type: "page", title: "Integrations", subtitle: "All integrations", section: "", href: "/integrations", }, { type: "page", title: "Feature Matrix", subtitle: "Compare features across integrations", section: "", href: "/matrix", }, { type: "page", title: "API Reference", subtitle: "CopilotKit components and hooks", section: "", href: "/reference", }, { type: "page", title: "AG-UI Overview", subtitle: "The Agent-User Interaction Protocol", section: "", href: "/ag-ui", }, ); // Track which scan roots exist so a misconfigured checkout produces a // loud empty-index rather than a tiny silent one. Missing all three // means the script is running outside a prepared tree (e.g. shell-docs // wasn't built into the expected layout) — fail the run so CI catches // the regression instead of shipping a crippled search modal. const scanDirsMissing: string[] = []; const scanDirsPresent: string[] = []; // CopilotKit Reference const refDir = path.join(CONTENT_ROOT, "content", "reference"); if (fs.existsSync(refDir)) { const refEntries = scanMdxDir(refDir, "/reference", "reference"); entries.push(...refEntries); console.log(` Reference: ${refEntries.length} entries`); scanDirsPresent.push(refDir); } else { console.warn( `[generate-search-index] scan dir missing: ${refDir} — reference entries will be empty`, ); scanDirsMissing.push(refDir); } // AG-UI docs — only index pages that are published in the AG-UI sidebar nav const AGUI_PUBLISHED_SLUGS = new Set([ "introduction", "agentic-protocols", "quickstart/applications", "quickstart/introduction", "quickstart/server", "quickstart/middleware", "quickstart/clients", "concepts/architecture", "concepts/events", "concepts/agents", "concepts/middleware", "concepts/messages", "concepts/reasoning", "concepts/state", "concepts/serialization", "concepts/tools", "concepts/capabilities", "concepts/generative-ui-specs", "drafts/overview", "drafts/multimodal-messages", "drafts/interrupts", "drafts/generative-ui", "drafts/meta-events", "tutorials/cursor", "tutorials/debugging", "development/updates", "development/roadmap", "development/contributing", "sdk/js/core/overview", "sdk/js/core/types", "sdk/js/core/multimodal-inputs", "sdk/js/core/events", "sdk/js/client/overview", "sdk/js/client/abstract-agent", "sdk/js/client/http-agent", "sdk/js/client/middleware", "sdk/js/client/subscriber", "sdk/js/client/compaction", "sdk/js/encoder", "sdk/js/proto", "sdk/python/core/overview", "sdk/python/core/types", "sdk/python/core/multimodal-inputs", "sdk/python/core/events", "sdk/python/encoder/overview", ]); const aguiDir = path.join(CONTENT_ROOT, "content", "ag-ui"); if (fs.existsSync(aguiDir)) { const aguiEntries = scanMdxDir( aguiDir, "/ag-ui", "ag-ui", AGUI_PUBLISHED_SLUGS, ); entries.push(...aguiEntries); console.log(` AG-UI: ${aguiEntries.length} entries`); scanDirsPresent.push(aguiDir); } else { console.warn( `[generate-search-index] scan dir missing: ${aguiDir} — ag-ui entries will be empty`, ); scanDirsMissing.push(aguiDir); } // CopilotKit Docs const docsDir = path.join(CONTENT_ROOT, "content", "docs"); if (fs.existsSync(docsDir)) { const docsEntries = scanMdxDir(docsDir, "/docs", "page").flatMap( normalizeDocsSearchEntry, ); entries.push(...docsEntries); console.log(` Docs: ${docsEntries.length} entries`); scanDirsPresent.push(docsDir); } else { console.warn( `[generate-search-index] scan dir missing: ${docsDir} — docs entries will be empty`, ); scanDirsMissing.push(docsDir); } if (scanDirsPresent.length === 0) { // Some build contexts (e.g. the `shell` Docker build) intentionally // omit the shell-docs content tree — they only need the static-pages // stub so the header search modal has something to render and links // resolve across to docs.showcase.copilotkit.ai. Warn loudly so a // misconfigured full build is still visible in logs, but don't fail. console.warn( `[generate-search-index] all scan directories missing — emitting static-pages stub only. Missing: ${scanDirsMissing.join(", ")}`, ); } // Write (dual-emit to shell-docs + shell) const json = JSON.stringify(entries, null, 2) + "\n"; for (const outputPath of OUTPUT_PATHS) { fs.mkdirSync(path.dirname(outputPath), { recursive: true }); fs.writeFileSync(outputPath, json); console.log(`\nSearch index: ${outputPath} (${entries.length} entries)`); } } main();