const assert = require("node:assert/strict"); const fs = require("node:fs"); const path = require("node:path"); const test = require("node:test"); const labeler = require("../pr-labeler.js"); const fixtures = require("./pr-labeler-fixtures.json"); const REPO_ROOT = path.resolve(__dirname, "../../.."); const LABELER_CONFIG_PATH = path.join(REPO_ROOT, ".github/labeler.yml"); const EXPECTED_LABELS = [ "cli-anything-hub", "cli-anything-skill", "documentation", "existing-cli-fix", "github-actions", "new-cli", ]; function normalizeLabels(labels) { return [...labels].sort(); } function sameLabels(left, right) { return JSON.stringify(left) === JSON.stringify(right); } function loadLabelerPatterns() { const patternsByLabel = new Map(); let currentLabel = null; for (const line of fs.readFileSync(LABELER_CONFIG_PATH, "utf8").split(/\r?\n/)) { const labelMatch = line.match(/^([a-z0-9-]+):$/); if (labelMatch) { currentLabel = labelMatch[1]; patternsByLabel.set(currentLabel, []); continue; } const patternMatch = line.match(/-\s+"([^"]+)"/); if (currentLabel && patternMatch) { patternsByLabel.get(currentLabel).push(patternMatch[1]); } } return patternsByLabel; } function globToRegex(pattern) { let regex = "^"; for (let index = 0; index < pattern.length; index += 1) { const char = pattern[index]; const next = pattern[index + 1]; const afterNext = pattern[index + 2]; if (char === "*" && next === "*" && afterNext === "/") { regex += "(?:.*/)?"; index += 2; continue; } if (char === "*" && next === "*") { regex += ".*"; index += 1; continue; } if (char === "*") { regex += "[^/]*"; continue; } regex += char.replace(/[|\\{}()[\]^$+?.]/g, "\\$&"); } regex += "$"; return new RegExp(regex); } function computePathLabels(files) { const labels = new Set(); const patternsByLabel = loadLabelerPatterns(); for (const [label, patterns] of patternsByLabel.entries()) { const regexes = patterns.map(globToRegex); if (files.some((file) => regexes.some((regex) => regex.test(file.filename)))) { labels.add(label); } } return labels; } function computeAllLabels(sample) { const labels = new Set(labeler.computeScriptLabels(sample.files, sample.title)); for (const pathLabel of computePathLabels(sample.files)) { labels.add(pathLabel); } return normalizeLabels(labels); } function summarizeMetrics(results) { const labels = new Set(EXPECTED_LABELS); for (const result of results) { for (const label of result.expected) labels.add(label); for (const label of result.predicted) labels.add(label); } const perLabel = {}; for (const label of labels) { let truePositive = 0; let falsePositive = 0; let falseNegative = 0; for (const result of results) { const expected = new Set(result.expected); const predicted = new Set(result.predicted); if (expected.has(label) && predicted.has(label)) truePositive += 1; if (!expected.has(label) && predicted.has(label)) falsePositive += 1; if (expected.has(label) && !predicted.has(label)) falseNegative += 1; } const precisionDenominator = truePositive + falsePositive; const recallDenominator = truePositive + falseNegative; perLabel[label] = { truePositive, falsePositive, falseNegative, precision: precisionDenominator === 0 ? 1 : truePositive / precisionDenominator, recall: recallDenominator === 0 ? 1 : truePositive / recallDenominator, }; } const exactMatches = results.filter((result) => sameLabels(result.predicted, result.expected)).length; return { exactAccuracy: exactMatches / results.length, perLabel, }; } test("real PR fixture label accuracy and recall", () => { const results = fixtures.map((sample) => ({ number: sample.number, title: sample.title, expected: normalizeLabels(sample.expected), predicted: computeAllLabels(sample), })); const mismatches = results.filter((result) => { return !sameLabels(result.predicted, result.expected); }); const metrics = summarizeMetrics(results); assert.deepStrictEqual(mismatches, [], JSON.stringify({mismatches, metrics}, null, 2)); assert.equal(metrics.exactAccuracy, 1); for (const [label, metric] of Object.entries(metrics.perLabel)) { assert.equal(metric.precision, 1, `${label} precision: ${JSON.stringify(metric)}`); assert.equal(metric.recall, 1, `${label} recall: ${JSON.stringify(metric)}`); } }); test("labeler config and script agree on the supported label set", () => { const configLabels = new Set(loadLabelerPatterns().keys()); const scriptLabels = new Set(Object.keys(labeler.LABELS)); for (const label of EXPECTED_LABELS) { assert(scriptLabels.has(label), `${label} must be creatable by pr-labeler.js`); } assert(configLabels.has("cli-anything-hub")); assert(configLabels.has("cli-anything-skill")); assert(configLabels.has("github-actions")); }); test("registry-only maintenance is not treated as a new CLI", () => { const labels = computeAllLabels({ title: "Update registry dates", files: [{filename: "registry.json", status: "modified"}], }); assert.deepStrictEqual(labels, ["cli-anything-hub"]); }); test("mixed README and harness changes are not documentation-only", () => { const labels = computeAllLabels({ title: "fix(blender): update docs and render behavior", files: [ {filename: "README.md", status: "modified"}, {filename: "blender/agent-harness/cli_anything/blender/core/render.py", status: "modified"}, ], }); assert.deepStrictEqual(labels, ["existing-cli-fix"]); }); test("harness README-only changes are documentation", () => { const labels = computeAllLabels({ title: "docs(firefly-iii): clarify setup notes", files: [{filename: "firefly-iii/agent-harness/README.md", status: "modified"}], }); assert.deepStrictEqual(labels, ["documentation"]); });