import fs from "node:fs/promises";
import path from "node:path";
const DEFAULT_INPUT = "docs/content/docs";
const DEFAULT_OUTPUT = "outputs/knowledge-base/DBX_knowledge_base_cn.md";
const DEFAULT_LANG = "cn";
await main();
async function main() {
const args = parseArgs(process.argv.slice(2));
const inputDir = path.resolve(args.input ?? DEFAULT_INPUT);
const outputFile = path.resolve(args.output ?? DEFAULT_OUTPUT);
const lang = args.lang ?? DEFAULT_LANG;
const fileSuffix = lang ? `.${lang}.mdx` : ".mdx";
const files = (await fs.readdir(inputDir))
.filter((name) => name.endsWith(fileSuffix))
.sort((a, b) => a.localeCompare(b));
if (files.length === 0) {
throw new Error(`No ${fileSuffix} files found in ${inputDir}`);
}
const documents = [];
let entryCount = 0;
for (const file of files) {
const sourcePath = path.join(inputDir, file);
const displayPath = path.relative(process.cwd(), sourcePath).replaceAll(path.sep, "/");
const raw = await fs.readFile(sourcePath, "utf8");
const { fields, body } = parseFrontmatter(raw);
const title = fields.title || file.replace(/\.mdx$/, "");
const description = fields.description || "";
const sections = [];
for (const section of splitSections(body)) {
const clean = normalizeMdx(section.lines.join("\n"));
if (!clean || clean.length < 20) continue;
const chunks = chunkText(clean).map((chunk, index, all) => {
const heading = all.length > 1 ? `${section.heading}(${index + 1}/${all.length})` : section.heading;
return {
heading,
question: questionFor(title, section.heading, chunk),
keywords: keywordsFor(title, section.heading, description, chunk),
answer: chunk,
};
});
entryCount += chunks.length;
sections.push(...chunks);
}
documents.push({ source: displayPath, title, description, sections });
}
await fs.mkdir(path.dirname(outputFile), { recursive: true });
await fs.writeFile(outputFile, renderMarkdown(documents), "utf8");
console.log(`Knowledge base written: ${path.relative(process.cwd(), outputFile)}`);
console.log(`Sources: ${files.length}`);
console.log(`Entries: ${entryCount}`);
}
function parseArgs(argv) {
const parsed = {};
for (let index = 0; index < argv.length; index += 1) {
const arg = argv[index];
if (arg === "--help" || arg === "-h") {
printHelp();
process.exit(0);
}
if (!arg.startsWith("--")) {
throw new Error(`Unexpected argument: ${arg}`);
}
const key = arg.slice(2);
const value = argv[index + 1];
if (!value || value.startsWith("--")) {
throw new Error(`Missing value for ${arg}`);
}
parsed[key] = value;
index += 1;
}
return parsed;
}
function printHelp() {
console.log(`Usage: node scripts/build-knowledge-base.mjs [options]
Options:
--input
MDX docs directory. Default: ${DEFAULT_INPUT}
--output Output Markdown path. Default: ${DEFAULT_OUTPUT}
--lang File language suffix, e.g. cn for *.cn.mdx. Default: ${DEFAULT_LANG}
`);
}
function parseFrontmatter(source) {
const match = source.match(/^---\n([\s\S]*?)\n---\n?/);
const fields = {};
if (!match) return { fields, body: source };
for (const line of match[1].split("\n")) {
const separator = line.indexOf(":");
if (separator === -1) continue;
const key = line.slice(0, separator).trim();
const value = line.slice(separator + 1).trim().replace(/^["']|["']$/g, "");
fields[key] = value;
}
return { fields, body: source.slice(match[0].length) };
}
function splitSections(body) {
const sections = [];
let current = { heading: "概览", lines: [] };
for (const line of body.split("\n")) {
const heading = line.match(/^(#{2,3})\s+(.+?)\s*$/);
if (heading) {
if (current.lines.join("\n").trim()) sections.push(current);
current = { heading: heading[2].replace(/`/g, "").trim(), lines: [] };
continue;
}
current.lines.push(line);
}
if (current.lines.join("\n").trim()) sections.push(current);
return sections;
}
function normalizeMdx(text) {
return text
.replace(/]*)>([\s\S]*?)<\/Card>/g, (_, attrs, inner) => {
const title = attrs.match(/title=["']([^"']+)["']/)?.[1] ?? "";
const href = attrs.match(/href=["']([^"']+)["']/)?.[1] ?? "";
return [title && `**${title}**`, normalizeMdx(inner), href && `参考链接:${href}`].filter(Boolean).join("\n\n");
})
.replace(/]*>([\s\S]*?)<\/Callout>/g, (_, inner) => normalizeMdx(inner))
.replace(/]*>([\s\S]*?)<\/Tab>/g, (_, inner) => normalizeMdx(inner))
.replace(/]*>([\s\S]*?)<\/Step>/g, (_, inner) => normalizeMdx(inner))
.replace(/<[^>\n]+>/g, "")
.replace(/\{\/\*[\s\S]*?\*\/\}/g, "")
.replace(/\[([^\]]+)\]\(([^)]+)\)/g, "$1($2)")
.replace(/^\s{0,6}#{1,6}\s+(.+)$/gm, "$1")
.replace(/^\s*import\s+.*$/gm, "")
.replace(/^\s*export\s+.*$/gm, "")
.replace(/[ \t]+\n/g, "\n")
.replace(/\n{3,}/g, "\n\n")
.trim();
}
function chunkText(text, maxLength = 2200) {
if (text.length <= maxLength) return [text];
const chunks = [];
let current = "";
for (const paragraph of text.split(/\n{2,}/)) {
const next = [current, paragraph].filter(Boolean).join("\n\n");
if (next.length > maxLength && current) {
chunks.push(current);
current = paragraph;
} else {
current = next;
}
}
if (current) chunks.push(current);
return chunks.flatMap((chunk) => {
if (chunk.length <= maxLength) return [chunk];
const pieces = [];
for (let index = 0; index < chunk.length; index += maxLength) {
pieces.push(chunk.slice(index, index + maxLength));
}
return pieces;
});
}
function questionFor(category, heading, text) {
if (heading === "概览") return `DBX 的${category}是什么?`;
if (/安装|配置|导入|导出|创建|连接|使用流程|快速开始|启动|构建|下载|打开|执行|添加|删除|更新/.test(heading)) {
return `DBX 中如何${heading.replace(/^(第一步|第二步|第三步|第四步|第五步):/, "")}?`;
}
if (/常见|问题|故障|错误|诊断|冲突|无法/.test(heading) || /失败|报错|检查/.test(text)) {
return `DBX ${heading}时应该检查什么?`;
}
if (/支持|覆盖|矩阵|类型|数据库|文件|供应商|快捷键|命令|端口/.test(heading)) {
return `DBX ${heading}支持哪些内容?`;
}
return `DBX 的${heading}是什么?`;
}
function keywordsFor(category, heading, description, text) {
const pool = `${category} ${heading} ${description} ${text}`;
const candidates = [
"DBX",
"AI",
"SQL",
"MCP",
"JDBC",
"SSH",
"Docker",
"CLI",
"MongoDB",
"Redis",
"MySQL",
"PostgreSQL",
"SQLite",
"DuckDB",
"ClickHouse",
"Oracle",
"SQL Server",
"Schema",
"ER",
"导入",
"导出",
"连接",
"表",
"字段",
"查询",
"快捷键",
"插件",
"驱动",
"配置",
"安全",
"数据传输",
"格式化",
"代码片段",
"隧道",
"生产环境",
];
const found = candidates.filter((word) => pool.includes(word));
return [...new Set([category, heading, ...found])].slice(0, 14).join(", ");
}
function renderMarkdown(documents) {
const lines = [
"# DBX 中文知识库",
"",
"> 从 `docs/content/docs/*.cn.mdx` 生成的纯 Markdown 知识库内容。",
"",
"## 来源索引",
"",
"| 来源文件 | 标题 | 描述 | 知识条数 |",
"| --- | --- | --- | ---: |",
];
for (const document of documents) {
lines.push(
`| ${escapeTableCell(document.source)} | ${escapeTableCell(document.title)} | ${escapeTableCell(document.description)} | ${document.sections.length} |`,
);
}
for (const document of documents) {
lines.push("", `## ${document.title}`, "");
if (document.description) lines.push(document.description, "");
lines.push(`来源:\`${document.source}\``, "");
for (const section of document.sections) {
lines.push(`### ${section.heading}`, "");
lines.push(`问题:${section.question}`, "");
lines.push(`关键词:${section.keywords}`, "");
lines.push("答案:", "");
lines.push(section.answer, "");
lines.push("---", "");
}
}
return `${lines.join("\n").replace(/\n{4,}/g, "\n\n\n").trim()}\n`;
}
function escapeTableCell(value) {
return String(value ?? "")
.replaceAll("\\", "\\\\")
.replaceAll("|", "\\|")
.replace(/\s+/g, " ")
.trim();
}