Files
composiohq--composio/docs/lib/source.ts
T
2026-07-13 12:38:34 +08:00

496 lines
18 KiB
TypeScript

import { docs, reference, examples, toolkits, changelog } from 'fumadocs-mdx:collections/server';
import { type InferPageType, loader, multiple } from 'fumadocs-core/source';
import { lucideIconsPlugin } from 'fumadocs-core/source/lucide-icons';
import { openapi, openapiV3 } from './openapi';
import { openapiSource, openapiPlugin } from 'fumadocs-openapi/server';
import { getGuardrails } from './llm-guardrails';
import { HIDDEN_API_TAGS } from './filter-api-version';
import { FILE_BUILDS } from './file-builds';
/**
* True if a reference URL belongs to an intentionally-hidden API tag
* (consumer, invite-codes) in either v3.1 or v3.0. These tags exist in the
* upstream OpenAPI spec but are hidden on our side. The page tree is filtered
* via `prepareTree` (lib/filter-api-version.ts); this mirror keeps the flat
* `getPages()` list (consumed by validate-links, llms.mdx, sitemap) in sync.
*/
function isHiddenReferenceUrl(url: string): boolean {
for (const tag of HIDDEN_API_TAGS) {
if (
url.startsWith(`/reference/api-reference/${tag}/`) ||
url === `/reference/api-reference/${tag}` ||
url.startsWith(`/reference/v3/api-reference/${tag}/`) ||
url === `/reference/v3/api-reference/${tag}`
) {
return true;
}
}
return false;
}
/**
* Transformer to set defaultOpen: true for specific folders in the reference sidebar.
*/
const defaultOpenTransformer = {
folder(node: { name: string; defaultOpen?: boolean }, folderPath: string) {
if (folderPath === 'api-reference' || folderPath === 'sdk-reference' || folderPath === 'v3/api-reference') {
return { ...node, defaultOpen: true };
}
return node;
},
};
export const source = loader({
baseUrl: '/docs',
source: docs.toFumadocsSource(),
plugins: [lucideIconsPlugin()],
});
// One combined reference source with both v3.1 and v3.0 OpenAPI pages.
// v3.1 at api-reference/, v3.0 at api-reference/v3/
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let _referenceSource: any = null;
// eslint-disable-next-line @typescript-eslint/no-explicit-any
let _openapiPagesPromise: Promise<any> | null = null;
async function getOpenapiPages() {
if (!_openapiPagesPromise) {
_openapiPagesPromise = Promise.all([
openapiSource(openapi, { groupBy: 'tag', baseDir: 'api-reference' }),
openapiSource(openapiV3, { groupBy: 'tag', baseDir: 'v3/api-reference' }),
]).catch((e) => {
// Don't permanently cache a failed load (e.g. a transient OpenAPI spec
// resolution error in a serverless instance). Clearing the memo lets the
// next request retry instead of re-throwing the same cached rejection.
_openapiPagesPromise = null;
throw e;
});
}
return _openapiPagesPromise;
}
export async function getReferenceSource() {
if (!_referenceSource) {
const [openapiLatest, openapiV3Pages] = await getOpenapiPages();
const loaded = loader({
baseUrl: '/reference',
source: multiple({
mdx: reference.toFumadocsSource(),
openapi: openapiLatest,
'openapi-v3': openapiV3Pages,
}),
plugins: [lucideIconsPlugin(), openapiPlugin()],
pageTree: {
// eslint-disable-next-line @typescript-eslint/no-explicit-any
transformers: [defaultOpenTransformer as any],
},
});
// Exclude intentionally-hidden API tags (consumer, invite-codes) from the
// flat page list so validate-links, llms.mdx, llms.txt, and sitemap skip
// their fumadocs-openapi operation pages. The sidebar tree is filtered
// separately via prepareTree (lib/filter-api-version.ts).
const originalGetPages = loaded.getPages.bind(loaded);
// eslint-disable-next-line @typescript-eslint/no-explicit-any
(loaded as any).getPages = (...args: Parameters<typeof originalGetPages>) =>
originalGetPages(...args).filter(
(page: { url: string }) => !isHiddenReferenceUrl(page.url),
);
_referenceSource = loaded;
}
return _referenceSource;
}
// Synchronous reference source for cases where OpenAPI isn't needed
export const referenceSource = loader({
baseUrl: '/reference',
source: reference.toFumadocsSource(),
plugins: [lucideIconsPlugin()],
});
export const examplesSource = loader({
baseUrl: '/examples',
source: examples.toFumadocsSource(),
plugins: [lucideIconsPlugin()],
});
export const toolkitsSource = loader({
baseUrl: '/toolkits',
source: toolkits.toFumadocsSource(),
plugins: [lucideIconsPlugin()],
});
export const changelogEntries = changelog;
export function getOgImageUrl(_section: string, _slugs: string[], title?: string, _description?: string): string {
const encodedTitle = encodeURIComponent(title ?? 'Composio Docs');
return `https://og.composio.dev/api/og?title=${encodedTitle}`;
}
/**
* Converts MDX content to clean markdown for AI agents.
* Strips JSX components and converts them to plain text equivalents.
*/
export function mdxToCleanMarkdown(content: string): string {
let result = content;
// Remove frontmatter
result = result.replace(/^---[\s\S]*?---\n*/m, '');
// Convert YouTube to link
result = result.replace(
/<YouTube\s+id="([^"]+)"\s+title="([^"]+)"\s*\/>/g,
'[Video: $2](https://youtube.com/watch?v=$1)'
);
// Convert Callout to blockquote - trim content to avoid empty lines
result = result.replace(
/<Callout[^>]*title="([^"]*)"[^>]*>([\s\S]*?)<\/Callout>/g,
(_, title, content) => `> **${title}**: ${content.trim()}`
);
result = result.replace(
/<Callout[^>]*>([\s\S]*?)<\/Callout>/g,
(_, content) => `> ${content.trim()}`
);
// Remove Cards wrapper before processing individual Card tags
result = result.replace(/<\/?Cards\b[^>]*>/g, '');
// Convert Card - handle multiline and various attribute orders
result = result.replace(
/<Card\b[\s\S]*?title="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?\/>/g,
'- [$1]($2): $3'
);
result = result.replace(
/<Card\b[\s\S]*?title="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Card>/g,
'- [$1]($2): $3'
);
result = result.replace(
/<Card\b[\s\S]*?href="([^"]*)"[\s\S]*?title="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Card>/g,
'- [$2]($1): $3'
);
result = result.replace(
/<ProviderCard[\s\S]*?name="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?languages=\{\[([^\]]*)\]\}[\s\S]*?\/>/g,
(_, name, href, langs) => `- [${name}](${href}) (${langs.replace(/"/g, '')})`
);
result = result.replace(/^[ \t]+(- \[)/gm, '$1');
result = result.replace(/<TabsList>[\s\S]*?<\/TabsList>/g, '');
result = result.replace(/<TabsTrigger[^>]*>[^<]*<\/TabsTrigger>/g, '');
result = result.replace(/<TabsContent[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/TabsContent>/g, '\n**$1:**\n$2');
result = result.replace(/<Tab[\s\S]*?value="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Tab>/g, '\n**$1:**\n$2');
result = result.replace(/<StepTitle>([\s\S]*?)<\/StepTitle>/g, (_, title) => {
const cleanTitle = title.replace(/^[\s#]*#\s*/, '').replace(/\s+$/, '').trim();
return cleanTitle ? `#### ${cleanTitle}` : '';
});
result = result.replace(/<Step>\s*###\s*(.+)/g, '#### $1');
result = result.replace(/<\/?Steps>/g, '');
result = result.replace(/<\/?Step>/g, '');
result = result.replace(/^(\s*#{1,6})\s*#\s+(.+)$/gm, '$1 $2');
result = result.replace(/^\s*#\s*$/gm, '');
result = result.replace(
/<FrameworkOption[\s\S]*?name="([^"]*)"[\s\S]*?>/g,
'\n## $1\n'
);
result = result.replace(/<\/FrameworkOption>/g, '');
const tabLabelMap: Record<string, string> = { native: 'Native Tools', mcp: 'MCP' };
result = result.replace(
/<IntegrationTabs[\s\S]*?tabs=\{\[([\s\S]*?)\]\}[\s\S]*?>/g,
(_, tabsContent: string) => {
const labelRegex = /value:\s*"([^"]+)"[\s\S]*?label:\s*"([^"]+)"/g;
let match;
while ((match = labelRegex.exec(tabsContent)) !== null) {
tabLabelMap[match[1]] = match[2];
}
return '\n> Choose your integration type · [Use this guide to decide](/docs/native-tools-vs-mcp)\n';
}
);
result = result.replace(
/<IntegrationTabs(?![^>]*tabs=)[\s\S]*?>/g,
'\n> Choose your integration type · [Use this guide to decide](/docs/native-tools-vs-mcp)\n'
);
result = result.replace(
/<IntegrationContent[\s\S]*?value="([^"]*)"[\s\S]*?>/g,
(_, value: string) => `\n### ${tabLabelMap[value] || value}\n`
);
result = result.replace(/<\/IntegrationContent>/g, '');
result = result.replace(
/<Accordion[\s\S]*?title="([^"]*)"[\s\S]*?>([\s\S]*?)<\/Accordion>/g,
'\n**$1**\n$2'
);
result = result.replace(
/<Figure[\s\S]*?src="([^"]*)"[\s\S]*?alt="([^"]*)"[\s\S]*?caption="([^"]*)"[\s\S]*?\/>/g,
'![$2]($1)\n*$3*'
);
result = result.replace(
/<Figure[\s\S]*?src="([^"]*)"[\s\S]*?alt="([^"]*)"[\s\S]*?\/>/g,
'![$2]($1)'
);
result = result.replace(
/<ToolTypeOption[\s\S]*?name="([^"]*)"[\s\S]*?>/g,
'\n### $1\n'
);
result = result.replace(/<\/ToolTypeOption>/g, '');
result = result.replace(
/<TemplateCard[\s\S]*?title="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?href="([^"]*)"[\s\S]*?\/>/g,
'- [$1]($3): $2'
);
result = result.replace(
/<TemplateCard[\s\S]*?href="([^"]*)"[\s\S]*?title="([^"]*)"[\s\S]*?description="([^"]*)"[\s\S]*?\/>/g,
'- [$2]($1): $3'
);
result = result.replace(
/<GlossaryTerm[\s\S]*?name="([^"]*)"[\s\S]*?>([\s\S]*?)<\/GlossaryTerm>/g,
(_, name, content) => `### ${name}\n\n${content.trim()}`
);
result = result.replace(
/<AIToolsBanner\s*\/>/g,
'### For AI tools\n\n' +
'**Skills:**\n' +
'```bash\nnpx skills add composiohq/skills\n```\n' +
'[Skills.sh](https://skills.sh/composiohq/skills/composio) · [GitHub](https://github.com/composiohq/skills)\n\n' +
'**CLI:**\n' +
'```bash\ncurl -fsSL https://composio.dev/install | bash\n```\n' +
'[CLI Reference](/docs/cli)\n\n' +
'**Context:**\n' +
'- [llms.txt](/llms.txt) — Documentation index with links\n' +
'- [llms-full.txt](/llms-full.txt) — Complete documentation in one file'
);
result = result.replace(
/<ConnectClientOption[^>]*\bname="([^"]*)"[^>]*>/g,
(_, name) => `## ${name}\n`
);
// FileBuildup renders an example's file growing step by step. The JSX can't
// serialize to markdown, so the .md an agent reads would otherwise lose every
// line of real code. Emit the actual source from the FILE_BUILDS registry:
// `<FileBuildup name="bot" step={2} />` -> the full file at that step;
// without `step` -> the final complete file.
result = result.replace(
/<FileBuildup\s+name="([^"]+)"(?:\s+step=\{(\d+)\})?\s*\/>/g,
(_, name: string, step?: string) => {
const build = FILE_BUILDS[name];
if (!build || !build.stages?.length) return '';
const lang = /\.tsx?$/.test(build.file)
? 'typescript'
: /\.py$/.test(build.file)
? 'python'
: '';
const idx = step ? Number(step) - 1 : build.stages.length - 1;
const stage = build.stages[idx];
if (!stage) return '';
const label = step ? ` — step ${step}: ${stage.title}` : ' — complete file';
return `\n**\`${build.file}\`${label}**\n\n\`\`\`${lang}\n${stage.code.trim()}\n\`\`\`\n`;
}
);
// RepoBrowser is an interactive file tree. The iMessage page intentionally
// shows a source slice while its public runnable fixture is still pending.
result = result.replace(
/<RepoBrowser\b(?=[^>]*\bsource="imessage")[^>]*\/>/g,
'\n> The iMessage code browser is an implementation slice, not a standalone fixture. The complete runnable project will be published in the Composio examples repo.\n'
);
// Point the existing Slack browser at its real repository.
result = result.replace(
/<RepoBrowser\b[^>]*\/>/g,
'\n> The complete project is on GitHub: [composio-slack-bot](https://github.com/ComposioHQ/composio-slack-bot).\n'
);
result = result.replace(/<\/?(ProviderGrid|Tabs|Frame|div|QuickstartFlow|IntegrationTabs|Accordions|ToolTypeFlow|ToolkitsLanding|TemplateGrid|Glossary|ConnectFlow|ConnectClientOption)[^>]*>/g, '');
result = result.replace(/<[A-Z][a-zA-Z]*[\s\S]*?\/>/g, '');
result = result.replace(/<\/?[A-Z][a-zA-Z]*[^>]*>/g, '');
const lines = result.split('\n');
const normalizedLines: string[] = [];
let inCodeBlock = false;
let codeBlockLines: string[] = [];
const flushCodeBlock = () => {
if (codeBlockLines.length > 0) {
const nonEmptyLines = codeBlockLines.filter(l => l.trim().length > 0);
const minIndent = nonEmptyLines.length > 0
? Math.min(...nonEmptyLines.map(l => l.match(/^(\s*)/)?.[1]?.length || 0))
: 0;
for (const codeLine of codeBlockLines) {
normalizedLines.push(codeLine.slice(minIndent));
}
codeBlockLines = [];
}
};
for (const line of lines) {
if (line.trim().startsWith('```')) {
if (inCodeBlock) {
flushCodeBlock();
inCodeBlock = false;
normalizedLines.push(line.trim());
} else {
inCodeBlock = true;
normalizedLines.push(line.trim());
}
} else if (inCodeBlock) {
codeBlockLines.push(line);
} else {
const trimmedLine = line.trimStart();
if (/^\s*[{}]\s*$/.test(line)) {
continue;
}
if (trimmedLine.match(/^[-*+]\s/) || trimmedLine.match(/^\d+\.\s/)) {
const leadingSpaces = line.length - trimmedLine.length;
const indentLevel = Math.floor(leadingSpaces / 2);
const normalizedIndent = ' '.repeat(Math.min(indentLevel, 4));
normalizedLines.push(normalizedIndent + trimmedLine);
} else {
normalizedLines.push(trimmedLine);
}
}
}
if (inCodeBlock) {
flushCodeBlock();
}
result = normalizedLines.join('\n');
result = result.replace(/\n{3,}/g, '\n\n');
result = stripTwoslashFromCodeBlocks(result);
return result.trim();
}
function stripTwoslashFromCodeBlocks(content: string): string {
return content.replace(/(```[\w]*\n)([\s\S]*?)(```)/g, (match, open, code, close) => {
let cleanCode = code;
cleanCode = cleanCode.replace(/^\/\/\s*---cut---.*\n?/gm, '');
cleanCode = cleanCode.replace(/^\/\/\s*@errors?:.*\n?/gm, '');
cleanCode = cleanCode.replace(/^\/\/\s*@noErrors.*\n?/gm, '');
cleanCode = cleanCode.replace(/^\/\/\s*@filename:.*\n?/gm, '');
cleanCode = cleanCode.replace(/^\/\/\s*@highlight.*\n?/gm, '');
cleanCode = cleanCode.replace(/^\/\/\s*\^[\?\!].*\n?/gm, '');
cleanCode = cleanCode.replace(/^\n+/, '');
return open + cleanCode + close;
});
}
export async function getLLMText(page: InferPageType<typeof source>, options?: { includeFooter?: boolean; includeGuardrails?: boolean }) {
const includeFooter = options?.includeFooter ?? true;
const includeGuardrails = options?.includeGuardrails ?? true;
if (typeof page.data.getText !== 'function') {
return `# ${page.data.title} (${page.url})
${page.data.description || ''}`;
}
let content: string | null = null;
try {
content = await page.data.getText('processed');
} catch (e) {
console.error('getText(processed) failed:', e);
try {
content = await page.data.getText('raw');
} catch (e2) {
console.error('getText(raw) also failed:', e2);
}
}
if (!content) {
return `# ${page.data.title} (${page.url})
${page.data.description || ''}`;
}
const mermaidRegex = /<Mermaid\s+chart="([\s\S]*?)"\s*\/>/g;
const segments: string[] = [];
const mermaidCharts: string[] = [];
let lastIndex = 0;
let match;
while ((match = mermaidRegex.exec(content)) !== null) {
segments.push(content.slice(lastIndex, match.index));
mermaidCharts.push(match[1]);
lastIndex = match.index + match[0].length;
}
segments.push(content.slice(lastIndex));
const cleanSegments = segments.map(s => mdxToCleanMarkdown(s));
let cleanContent = cleanSegments[0];
for (let i = 0; i < mermaidCharts.length; i++) {
const chart = mermaidCharts[i].replace(/&#x22;/g, '"').replace(/&#x27;/g, "'").replace(/&amp;/g, '&');
cleanContent += `\n\n\`\`\`mermaid\n${chart}\n\`\`\`\n\n${cleanSegments[i + 1]}`;
}
const footer = includeFooter
? `\n\n---\n\n📚 **More documentation:** [View all docs](https://docs.composio.dev/llms.txt) | [Glossary](https://docs.composio.dev/llms.mdx/reference/glossary) | [Examples](https://docs.composio.dev/llms.mdx/examples) | [API Reference](https://docs.composio.dev/llms.mdx/reference)`
: '';
// Legacy pages (frontmatter `legacy: true`) document point-in-time migrations
// and may show outdated APIs. Mark the .md so an agent reading it knows, and
// skip the "enforce the CURRENT patterns" guardrail block — appending it to a
// legacy guide contradicts the guide's own (older) content.
const isLegacy = page.data.legacy === true;
const written = page.data.written;
const topNote = isLegacy
? `\n> **Legacy${written ? ` · written ${written}` : ''}.** This is a point-in-time migration/legacy guide and may describe outdated APIs. For current guidance, see https://docs.composio.dev.\n`
: written
? `\n> _Written ${written}._\n`
: '';
const guardrails =
includeGuardrails && !isLegacy ? getGuardrails(page.data.llmGuardrails) : '';
return `# ${page.data.title} (${page.url})
${topNote}
${cleanContent}${footer}${guardrails}`;
}
export function formatDate(dateStr: string): string {
return new Date(`${dateStr}T12:00:00`).toLocaleDateString('en-US', {
year: 'numeric',
month: 'short',
day: 'numeric',
});
}
const DATE_REGEX = /^\d{4}-\d{2}-\d{2}$/;
function validateDateFormat(dateStr: string): void {
if (!DATE_REGEX.test(dateStr)) {
throw new Error(
`Invalid date format: "${dateStr}". Expected YYYY-MM-DD (e.g., "2025-12-29")`
);
}
}
export function dateToChangelogUrl(dateStr: string): string {
validateDateFormat(dateStr);
const [year, month, day] = dateStr.split('-');
return `/docs/changelog/${year}/${month}/${day}`;
}
export function dateToSlug(dateStr: string): string[] {
validateDateFormat(dateStr);
const [year, month, day] = dateStr.split('-');
return [year, month, day];
}
export function slugToDate(slug: string[]): string | null {
if (slug.length !== 3) return null;
const [year, month, day] = slug;
return `${year}-${month}-${day}`;
}