wehub-resource-sync
337 lines
9.1 KiB
TypeScript
337 lines
9.1 KiB
TypeScript
import * as fs from "node:fs";
|
|
import * as path from "node:path";
|
|
import { unified } from "unified";
|
|
import remarkParse from "remark-parse";
|
|
import remarkMdx from "remark-mdx";
|
|
import { visit } from "unist-util-visit";
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Types
|
|
// ---------------------------------------------------------------------------
|
|
|
|
interface CodeBlock {
|
|
lang: string;
|
|
title: string;
|
|
doctest: string;
|
|
code: string;
|
|
line: number;
|
|
sourceFile: string;
|
|
}
|
|
|
|
interface ManifestEntry {
|
|
id: string;
|
|
file: string;
|
|
lang: string;
|
|
category: string;
|
|
source: string;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Config
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const DOCS_DIR = path.resolve(
|
|
__dirname,
|
|
"../../showcase/shell-docs/src/content",
|
|
);
|
|
const OUTPUT_DIR = path.resolve(__dirname, "../../.doctest-output");
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// AST Extraction
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const parser = unified().use(remarkParse).use(remarkMdx);
|
|
|
|
/**
|
|
* Strip common leading whitespace from all lines of a code block.
|
|
* Handles indented code blocks inside JSX (Tabs, If, etc.) that
|
|
* preserve the JSX indentation in the extracted code.
|
|
*/
|
|
function stripCommonIndent(code: string): string {
|
|
const lines = code.split("\n");
|
|
const nonEmptyLines = lines.filter((l) => l.trim().length > 0);
|
|
if (nonEmptyLines.length === 0) return code;
|
|
|
|
const minIndent = Math.min(
|
|
...nonEmptyLines.map((l) => l.match(/^(\s*)/)![1].length),
|
|
);
|
|
if (minIndent === 0) return code;
|
|
|
|
return lines.map((l) => l.slice(minIndent)).join("\n");
|
|
}
|
|
|
|
/**
|
|
* Parse the meta string from a code fence to extract key-value attributes.
|
|
*
|
|
* Handles formats like:
|
|
* python title="main.py" doctest="server"
|
|
* typescript title="server.ts" doctest="component"
|
|
*/
|
|
export function parseMeta(meta: string): Record<string, string> {
|
|
const attrs: Record<string, string> = {};
|
|
// Match key="value" or key='value'
|
|
const regex = /(\w+)=["']([^"']+)["']/g;
|
|
let match: RegExpExecArray | null;
|
|
while ((match = regex.exec(meta)) !== null) {
|
|
attrs[match[1]] = match[2];
|
|
}
|
|
return attrs;
|
|
}
|
|
|
|
/**
|
|
* Extract all code blocks with a doctest attribute from an MDX file.
|
|
*/
|
|
export function extractFromMdx(
|
|
content: string,
|
|
sourceFile: string,
|
|
): CodeBlock[] {
|
|
const blocks: CodeBlock[] = [];
|
|
|
|
let tree: ReturnType<typeof parser.parse>;
|
|
try {
|
|
tree = parser.parse(content);
|
|
} catch {
|
|
// Some MDX files have JSX constructs that trip the parser.
|
|
// Fall back to a regex-based extraction for resilience.
|
|
return extractFromMdxFallback(content, sourceFile);
|
|
}
|
|
|
|
visit(tree, "code", (node: any) => {
|
|
const lang = node.lang || "";
|
|
const meta = node.meta || "";
|
|
const attrs = parseMeta(meta);
|
|
|
|
if (!attrs.doctest) return;
|
|
|
|
const line =
|
|
node.position && node.position.start ? node.position.start.line : 0;
|
|
|
|
blocks.push({
|
|
lang,
|
|
title: attrs.title || `snippet.${langToExt(lang)}`,
|
|
doctest: attrs.doctest,
|
|
code: stripCommonIndent(node.value),
|
|
line,
|
|
sourceFile,
|
|
});
|
|
});
|
|
|
|
return blocks;
|
|
}
|
|
|
|
/**
|
|
* Regex-based fallback for MDX files that trip the remark-mdx parser.
|
|
* Only extracts code blocks with doctest attributes — less precise on
|
|
* position, but sufficient for our purposes.
|
|
*/
|
|
function extractFromMdxFallback(
|
|
content: string,
|
|
sourceFile: string,
|
|
): CodeBlock[] {
|
|
const blocks: CodeBlock[] = [];
|
|
const lines = content.split("\n");
|
|
|
|
let inBlock = false;
|
|
let blockLang = "";
|
|
let blockMeta = "";
|
|
let blockLines: string[] = [];
|
|
let blockStart = 0;
|
|
|
|
for (let i = 0; i < lines.length; i++) {
|
|
const trimmed = lines[i].trimStart();
|
|
|
|
if (!inBlock && /^```(\w+)(.*)$/.test(trimmed)) {
|
|
const match = trimmed.match(/^```(\w+)(.*)$/);
|
|
if (match) {
|
|
blockLang = match[1];
|
|
blockMeta = match[2];
|
|
blockLines = [];
|
|
blockStart = i + 1;
|
|
inBlock = true;
|
|
}
|
|
} else if (inBlock && /^```\s*$/.test(trimmed)) {
|
|
const attrs = parseMeta(blockMeta);
|
|
if (attrs.doctest) {
|
|
blocks.push({
|
|
lang: blockLang,
|
|
title: attrs.title || `snippet.${langToExt(blockLang)}`,
|
|
doctest: attrs.doctest,
|
|
code: stripCommonIndent(blockLines.join("\n")),
|
|
line: blockStart,
|
|
sourceFile,
|
|
});
|
|
}
|
|
inBlock = false;
|
|
} else if (inBlock) {
|
|
blockLines.push(lines[i]);
|
|
}
|
|
}
|
|
|
|
return blocks;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Helpers
|
|
// ---------------------------------------------------------------------------
|
|
|
|
function langToExt(lang: string): string {
|
|
switch (lang) {
|
|
case "python":
|
|
return "py";
|
|
case "typescript":
|
|
case "tsx":
|
|
return "ts";
|
|
case "javascript":
|
|
case "jsx":
|
|
return "js";
|
|
default:
|
|
return lang || "txt";
|
|
}
|
|
}
|
|
|
|
function slugify(filePath: string): string {
|
|
return filePath
|
|
.replace(/\.mdx$/, "")
|
|
.replace(/[/\\]/g, "-")
|
|
.replace(/[^a-zA-Z0-9-]/g, "");
|
|
}
|
|
|
|
/**
|
|
* Walk a directory tree and return all .mdx files.
|
|
*/
|
|
function findMdxFiles(dir: string): string[] {
|
|
const results: string[] = [];
|
|
|
|
function walk(current: string) {
|
|
const entries = fs.readdirSync(current, { withFileTypes: true });
|
|
for (const entry of entries) {
|
|
const full = path.join(current, entry.name);
|
|
if (entry.isDirectory()) {
|
|
if (entry.name.startsWith(".") || entry.name === "node_modules")
|
|
continue;
|
|
walk(full);
|
|
} else if (entry.name.endsWith(".mdx")) {
|
|
results.push(full);
|
|
}
|
|
}
|
|
}
|
|
|
|
walk(dir);
|
|
return results.sort();
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Output generation
|
|
// ---------------------------------------------------------------------------
|
|
|
|
/**
|
|
* Group extracted blocks by page slug and title, then write to output dir.
|
|
* Blocks sharing the same title within a page are concatenated into one file.
|
|
*/
|
|
export function writeExtractedBlocks(
|
|
blocks: CodeBlock[],
|
|
outputDir: string,
|
|
docsDir: string,
|
|
): ManifestEntry[] {
|
|
const manifest: ManifestEntry[] = [];
|
|
|
|
// Group by (page slug, title)
|
|
const grouped = new Map<string, CodeBlock[]>();
|
|
for (const block of blocks) {
|
|
const rel = path.relative(docsDir, block.sourceFile);
|
|
const slug = slugify(rel);
|
|
const key = `${slug}/${block.title}`;
|
|
const existing = grouped.get(key) || [];
|
|
existing.push(block);
|
|
grouped.set(key, existing);
|
|
}
|
|
|
|
for (const [key, groupBlocks] of grouped) {
|
|
const slug = key.split("/")[0];
|
|
const title = groupBlocks[0].title;
|
|
const dir = path.join(outputDir, slug);
|
|
|
|
fs.mkdirSync(dir, { recursive: true });
|
|
|
|
// Concatenate code from all blocks sharing this title
|
|
const code = groupBlocks.map((b) => b.code).join("\n\n");
|
|
const filePath = path.join(dir, title);
|
|
fs.writeFileSync(filePath, code, "utf-8");
|
|
|
|
// Copy doctest.json sidecar if it exists
|
|
const sidecarPath = path.join(
|
|
path.dirname(groupBlocks[0].sourceFile),
|
|
"doctest.json",
|
|
);
|
|
const destSidecar = path.join(dir, "doctest.json");
|
|
if (fs.existsSync(sidecarPath) && !fs.existsSync(destSidecar)) {
|
|
fs.copyFileSync(sidecarPath, destSidecar);
|
|
}
|
|
|
|
const firstBlock = groupBlocks[0];
|
|
const relSource = path.relative(
|
|
path.resolve(docsDir, ".."),
|
|
firstBlock.sourceFile,
|
|
);
|
|
|
|
const id = `${slug}-${title.replace(/[^a-zA-Z0-9]/g, "-")}`;
|
|
|
|
manifest.push({
|
|
id,
|
|
file: `${slug}/${title}`,
|
|
lang: firstBlock.lang,
|
|
category: firstBlock.doctest,
|
|
source: `${relSource}:${firstBlock.line}`,
|
|
});
|
|
}
|
|
|
|
return manifest;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Main
|
|
// ---------------------------------------------------------------------------
|
|
|
|
export function extract(
|
|
docsDir: string = DOCS_DIR,
|
|
outputDir: string = OUTPUT_DIR,
|
|
): ManifestEntry[] {
|
|
// Clean output dir
|
|
if (fs.existsSync(outputDir)) {
|
|
fs.rmSync(outputDir, { recursive: true });
|
|
}
|
|
fs.mkdirSync(outputDir, { recursive: true });
|
|
|
|
const files = findMdxFiles(docsDir);
|
|
const allBlocks: CodeBlock[] = [];
|
|
|
|
for (const file of files) {
|
|
const content = fs.readFileSync(file, "utf-8");
|
|
const blocks = extractFromMdx(content, file);
|
|
allBlocks.push(...blocks);
|
|
}
|
|
|
|
const manifest = writeExtractedBlocks(allBlocks, outputDir, docsDir);
|
|
|
|
// Write manifest
|
|
const manifestPath = path.join(outputDir, "manifest.json");
|
|
fs.writeFileSync(manifestPath, JSON.stringify(manifest, null, 2), "utf-8");
|
|
|
|
console.log(`Extracted ${manifest.length} doctest snippet(s):`);
|
|
for (const entry of manifest) {
|
|
console.log(` ${entry.id} [${entry.category}] ${entry.source}`);
|
|
}
|
|
|
|
return manifest;
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// CLI entry point
|
|
// ---------------------------------------------------------------------------
|
|
|
|
const isDirectRun = typeof require !== "undefined" && require.main === module;
|
|
|
|
if (isDirectRun) {
|
|
extract();
|
|
}
|