frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: expand family-based trigger regression suite

Browse Source
This commit is contained in:
yaojingang
2026-03-31 21:37:01 +08:00
parent 0aa571554b
commit 51a479d12f
16 changed files with 1756 additions and 486 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+5 -2
View File
@@ -1,6 +1,6 @@
PYTHON ?= python3
.PHONY: eval eval-suite package-check package-failure-check snapshot-check validate lint test clean
.PHONY: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint test clean
eval:
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
@@ -8,6 +8,9 @@ eval:
eval-suite:
$(PYTHON) scripts/run_eval_suite.py
failure-regression-check:
$(PYTHON) tests/verify_failure_regressions.py
package-check:
$(PYTHON) scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --output-dir dist --zip
@@ -23,7 +26,7 @@ validate:
lint:
$(PYTHON) scripts/lint_skill.py .
test: eval eval-suite package-check package-failure-check snapshot-check validate lint
test: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint
clean:
rm -rf dist tests/tmp tests/tmp_snapshot
README.md
+5 -3
View File
@@ -48,9 +48,10 @@ make test
- `make test` passes locally
- semantic trigger eval: `0` false positives, `0` false negatives on the current regression set
- eval suite: train / dev / holdout all pass
- eval suite: train / dev / holdout all pass, with family-based reporting
- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks
- packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected
- failure library regressions: anti-pattern families pass automated checks
## What It Does
@@ -111,7 +112,7 @@ Long-form material that should not bloat the main skill file. This includes desi
Utility scripts that make the meta-skill operational:
- `trigger_eval.py`: evaluates trigger descriptions with semantic intent concepts, explicit exclusions, and near-neighbor prompts
- `run_eval_suite.py`: runs train/dev/holdout trigger suites and fails if aggregate regressions appear
- `run_eval_suite.py`: runs train/dev/holdout trigger suites, reports family-level regressions, and fails if aggregate regressions appear
- `context_sizer.py`: estimates context weight and warns when the initial load gets too large
- `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation
- `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain
@@ -135,7 +136,7 @@ Continuous integration entrypoint that runs the full local regression suite on p
- Train/dev/holdout trigger suites now separate iterative tuning from final verification.
- Packaging validation now uses explicit contracts and YAML parsing, but it is still a lightweight local validation layer rather than a full platform integration suite.
- `evals/failure-cases.md` captures known weak spots that should remain part of regression checks.
- `failures/` captures reusable anti-pattern writeups for routing, packaging, and authoring failures.
- `failures/` captures reusable anti-pattern writeups and machine-runnable failure cases for routing, packaging, and authoring failures.
- `tests/verify_packager_failures.py` checks that invalid metadata, invalid YAML, and unsupported targets fail clearly.
### `templates/`
@@ -207,6 +208,7 @@ This project is best for:
- Examples: [examples/README.md](examples/README.md)
- Evals: [evals/README.md](evals/README.md)
- Failure library: [failures/README.md](failures/README.md)
- Failure regression check: [verify_failure_regressions.py](tests/verify_failure_regressions.py)
- Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md)
- Platform capability matrix: [references/platform-capability-matrix.md](references/platform-capability-matrix.md)
- Failure fixtures: [tests/fixtures](tests/fixtures)
evals/README.md
+3 -1
View File
@@ -4,7 +4,7 @@ This directory makes trigger quality and packaging quality more reproducible.
Contents:
- `trigger_cases.json`: positive, negative, and near-neighbor prompts
- `trigger_cases.json`: full regression set with `family` labels
- `train/`, `dev/`, `holdout/`: split trigger suites for iterative tuning and final verification
- `semantic_config.json`: local semantic-intent concepts, exclusions, and weights
- `baseline_description.txt`: intentionally weaker trigger description
@@ -19,6 +19,7 @@ Use:
python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json
python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
python3 scripts/run_eval_suite.py
python3 tests/verify_failure_regressions.py
python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip
python3 tests/verify_packager_failures.py
```
@@ -32,4 +33,5 @@ Regression scope now includes:
- mixed-intent negatives
- explicit "do not build a skill" negatives
- semantic exclusion cases such as one-off, document-only, and future-outline prompts
- family-based reporting across workflow-to-skill, iterate-existing-skill, document-only, one-off, and future-outline cases
- holdout verification
evals/dev/trigger_cases.json
+60 -9
View File
@@ -10,18 +10,69 @@
"summarize"
],
"should_trigger": [
"Convert this operations checklist into a reusable skill.",
"Add trigger evals to this skill before sharing it with the team.",
"We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks."
{
"text": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill"
},
{
"text": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill"
},
{
"text": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset"
},
{
"text": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset"
},
{
"text": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill"
}
],
"should_not_trigger": [
"Give me ideas for improving our process.",
"Review this note and tell me what it means in plain English.",
"Help me brainstorm several ways to improve packaging, but do not generate any skill files."
{
"text": "Give me ideas for improving our process.",
"family": "brainstorm_only"
},
{
"text": "Review this note and tell me what it means in plain English.",
"family": "explain_only"
},
{
"text": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only"
},
{
"text": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only"
},
{
"text": "Summarize this workflow and list the main points only.",
"family": "summary_only"
}
],
"near_neighbor": [
"Review this process note and explain it, no packaging needed.",
"Turn this into a checklist and template, but stop short of making a full skill.",
"Package this explanation as a document, not as an agent skill."
{
"text": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package"
},
{
"text": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill"
},
{
"text": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill"
},
{
"text": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build"
},
{
"text": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill"
}
]
}
evals/holdout/trigger_cases.json
+32 -5
View File
@@ -10,14 +10,41 @@
"summarize"
],
"should_trigger": [
"Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution."
{
"text": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset"
},
{
"text": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill"
},
{
"text": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill"
}
],
"should_not_trigger": [
"I pasted a long process description below, but I only want a summary, not a reusable skill."
{
"text": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only"
},
{
"text": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only"
}
],
"near_neighbor": [
"Help me shape an idea before we decide whether to build a skill.",
"Create an outline for a possible future skill, but do not build the skill yet."
{
"text": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build"
},
{
"text": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build"
},
{
"text": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill"
}
]
}
evals/sample_trigger_report.json
+1205 -417
View File
File diff suppressed because it is too large Load Diff
evals/semantic_config.json
+10
View File
@@ -108,6 +108,8 @@
"explain what",
"just explain",
"explain it",
"explain this",
"in simpler language",
"tell me what it means",
"plain english"
]
@@ -162,9 +164,14 @@
"do not generate any skill files",
"not a reusable skill",
"not as an agent skill",
"not an agent skill",
"without turning them into an agent skill",
"do not turn it into a skill",
"do not package it",
"stop short of making a full skill",
"not a skill yet",
"plain note",
"keep it as a plain note",
"no packaging needed"
]
},
@@ -174,6 +181,9 @@
"phrases": [
"readme",
"document",
"document package",
"reusable-looking document",
"wiki",
"article",
"blog title",
"headline",
evals/train/trigger_cases.json
+88 -17
View File
@@ -10,26 +10,97 @@
"summarize"
],
"should_trigger": [
"Create a skill from this repeated workflow.",
"Improve this skill description and add evals.",
"Turn this runbook into a reusable agent skill.",
"Package this skill for team reuse.",
"Refactor this prompt into a proper skill package.",
"Create a meta-skill for packaging internal workflows."
{
"text": "Create a skill from this repeated workflow.",
"family": "workflow_to_skill"
},
{
"text": "Turn this runbook into a reusable agent skill.",
"family": "workflow_to_skill"
},
{
"text": "Convert this process note into a reusable skill package for the team.",
"family": "workflow_to_skill"
},
{
"text": "Improve this skill description and add evals.",
"family": "iterate_existing_skill"
},
{
"text": "Refactor this prompt into a proper skill package.",
"family": "iterate_existing_skill"
},
{
"text": "Package this skill for team reuse.",
"family": "package_for_team"
},
{
"text": "Create a meta-skill for packaging internal workflows.",
"family": "meta_skill_creation"
},
{
"text": "Standardize this recurring workflow as a shareable skill package with references.",
"family": "package_for_team"
}
],
"should_not_trigger": [
"Explain what a workflow is.",
"Write a product headline for this landing page.",
"Summarize this random note.",
"Just explain what a skill is.",
"Draft a blog title for this article.",
"Translate this README into Japanese."
{
"text": "Explain what a workflow is.",
"family": "explain_only"
},
{
"text": "Just explain what a skill is.",
"family": "explain_only"
},
{
"text": "Summarize this random note.",
"family": "summary_only"
},
{
"text": "Translate this README into Japanese.",
"family": "translate_only"
},
{
"text": "Write a product headline for this landing page.",
"family": "document_only"
},
{
"text": "Draft a blog title for this article.",
"family": "document_only"
},
{
"text": "Rewrite this paragraph more clearly, but do not package anything.",
"family": "document_only"
},
{
"text": "Explain the difference between a runbook and a workflow.",
"family": "explain_only"
}
],
"near_neighbor": [
"Create a one-off prompt for this task.",
"Help me brainstorm process ideas without building a skill.",
"Improve this README but do not turn it into a skill.",
"Make a checklist for this task, but not a reusable skill.",
"Write a custom answer for this request without creating a skill package."
{
"text": "Create a one-off prompt for this task.",
"family": "one_off_vs_reusable"
},
{
"text": "Write a custom answer for this request without creating a skill package.",
"family": "one_off_vs_reusable"
},
{
"text": "Help me brainstorm process ideas without building a skill.",
"family": "brainstorm_vs_build"
},
{
"text": "Improve this README but do not turn it into a skill.",
"family": "document_export_vs_agent_skill"
},
{
"text": "Make a checklist for this task, but not a reusable skill.",
"family": "partial_scaffold_not_full_skill"
},
{
"text": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.",
"family": "partial_scaffold_not_full_skill"
}
]
}
evals/trigger_cases.json
+180 -31
View File
@@ -10,40 +10,189 @@
"summarize"
],
"should_trigger": [
"Create a skill from this repeated workflow.",
"Improve this skill description and add evals.",
"Turn this runbook into a reusable agent skill.",
"Package this skill for team reuse.",
"Convert this operations checklist into a reusable skill.",
"Refactor this prompt into a proper skill package.",
"Add trigger evals to this skill before sharing it with the team.",
"Create a meta-skill for packaging internal workflows.",
"We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution."
{
"text": "Create a skill from this repeated workflow.",
"family": "workflow_to_skill"
},
{
"text": "Turn this runbook into a reusable agent skill.",
"family": "workflow_to_skill"
},
{
"text": "Convert this process note into a reusable skill package for the team.",
"family": "workflow_to_skill"
},
{
"text": "Improve this skill description and add evals.",
"family": "iterate_existing_skill"
},
{
"text": "Refactor this prompt into a proper skill package.",
"family": "iterate_existing_skill"
},
{
"text": "Package this skill for team reuse.",
"family": "package_for_team"
},
{
"text": "Create a meta-skill for packaging internal workflows.",
"family": "meta_skill_creation"
},
{
"text": "Standardize this recurring workflow as a shareable skill package with references.",
"family": "package_for_team"
},
{
"text": "Convert this operations checklist into a reusable skill.",
"family": "workflow_to_skill"
},
{
"text": "Add trigger evals to this skill before sharing it with the team.",
"family": "iterate_existing_skill"
},
{
"text": "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.",
"family": "complex_multi_asset"
},
{
"text": "Turn these workflow fragments, transcripts, and prompts into one production-ready skill with scripts and references.",
"family": "complex_multi_asset"
},
{
"text": "Tighten the trigger boundary on this existing skill and prepare it for team distribution.",
"family": "iterate_existing_skill"
},
{
"text": "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.",
"family": "complex_multi_asset"
},
{
"text": "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution.",
"family": "iterate_existing_skill"
},
{
"text": "Turn this recurring onboarding process into a reusable skill library entry with validation notes.",
"family": "workflow_to_skill"
}
],
"should_not_trigger": [
"Explain what a workflow is.",
"Write a product headline for this landing page.",
"Summarize this random note.",
"Just explain what a skill is.",
"Draft a blog title for this article.",
"Translate this README into Japanese.",
"Give me ideas for improving our process.",
"I pasted a long process description below, but I only want a summary, not a reusable skill.",
"Review this note and tell me what it means in plain English.",
"Help me brainstorm several ways to improve packaging, but do not generate any skill files."
{
"text": "Explain what a workflow is.",
"family": "explain_only"
},
{
"text": "Just explain what a skill is.",
"family": "explain_only"
},
{
"text": "Summarize this random note.",
"family": "summary_only"
},
{
"text": "Translate this README into Japanese.",
"family": "translate_only"
},
{
"text": "Write a product headline for this landing page.",
"family": "document_only"
},
{
"text": "Draft a blog title for this article.",
"family": "document_only"
},
{
"text": "Rewrite this paragraph more clearly, but do not package anything.",
"family": "document_only"
},
{
"text": "Explain the difference between a runbook and a workflow.",
"family": "explain_only"
},
{
"text": "Give me ideas for improving our process.",
"family": "brainstorm_only"
},
{
"text": "Review this note and tell me what it means in plain English.",
"family": "explain_only"
},
{
"text": "Help me brainstorm several ways to improve packaging, but do not generate any skill files.",
"family": "brainstorm_only"
},
{
"text": "Translate these notes into French and keep the structure exactly as-is.",
"family": "translate_only"
},
{
"text": "Summarize this workflow and list the main points only.",
"family": "summary_only"
},
{
"text": "I pasted a long process description below, but I only want a summary, not a reusable skill.",
"family": "summary_only"
},
{
"text": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_only"
}
],
"near_neighbor": [
"Create a one-off prompt for this task.",
"Help me brainstorm process ideas without building a skill.",
"Improve this README but do not turn it into a skill.",
"Make a checklist for this task, but not a reusable skill.",
"Review this process note and explain it, no packaging needed.",
"Help me shape an idea before we decide whether to build a skill.",
"Write a custom answer for this request without creating a skill package.",
"Turn this into a checklist and template, but stop short of making a full skill.",
"Package this explanation as a document, not as an agent skill.",
"Create an outline for a possible future skill, but do not build the skill yet."
{
"text": "Create a one-off prompt for this task.",
"family": "one_off_vs_reusable"
},
{
"text": "Write a custom answer for this request without creating a skill package.",
"family": "one_off_vs_reusable"
},
{
"text": "Help me brainstorm process ideas without building a skill.",
"family": "brainstorm_vs_build"
},
{
"text": "Improve this README but do not turn it into a skill.",
"family": "document_export_vs_agent_skill"
},
{
"text": "Make a checklist for this task, but not a reusable skill.",
"family": "partial_scaffold_not_full_skill"
},
{
"text": "Create a reusable-looking checklist, but keep it as a plain note instead of a skill.",
"family": "partial_scaffold_not_full_skill"
},
{
"text": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package"
},
{
"text": "Turn this into a checklist and template, but stop short of making a full skill.",
"family": "partial_scaffold_not_full_skill"
},
{
"text": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill"
},
{
"text": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build"
},
{
"text": "Polish these notes into a reusable-looking document without turning them into an agent skill.",
"family": "document_export_vs_agent_skill"
},
{
"text": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build"
},
{
"text": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build"
},
{
"text": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill"
}
]
}
failures/README.md
+2
View File
@@ -14,3 +14,5 @@ Current case library:
- `explain-not-package`
- `document-export-vs-agent-skill`
- `future-outline-vs-build`
Each family now includes a `cases.json` file so the anti-pattern can run in automated regression instead of living only as prose.
failures/document-export-vs-agent-skill/cases.json
+24
View File
@@ -0,0 +1,24 @@
{
"recommended_threshold": 0.33,
"should_trigger": [],
"should_not_trigger": [
{
"text": "Rewrite this paragraph more clearly, but do not package anything.",
"family": "document_only"
}
],
"near_neighbor": [
{
"text": "Improve this README but do not turn it into a skill.",
"family": "document_export_vs_agent_skill"
},
{
"text": "Package this explanation as a document, not as an agent skill.",
"family": "document_export_vs_agent_skill"
},
{
"text": "Turn this into a document package for the wiki, not an agent skill.",
"family": "document_export_vs_agent_skill"
}
]
}
failures/explain-not-package/cases.json
+24
View File
@@ -0,0 +1,24 @@
{
"recommended_threshold": 0.33,
"should_trigger": [],
"should_not_trigger": [
{
"text": "Explain what a workflow is.",
"family": "explain_only"
},
{
"text": "Review this note and tell me what it means in plain English.",
"family": "explain_only"
}
],
"near_neighbor": [
{
"text": "Review this process note and explain it, no packaging needed.",
"family": "explain_not_package"
},
{
"text": "Explain this release checklist in simpler language; do not package it.",
"family": "explain_not_package"
}
]
}
failures/future-outline-vs-build/cases.json
+19
View File
@@ -0,0 +1,19 @@
{
"recommended_threshold": 0.33,
"should_trigger": [],
"should_not_trigger": [],
"near_neighbor": [
{
"text": "Help me shape an idea before we decide whether to build a skill.",
"family": "future_outline_vs_build"
},
{
"text": "Create an outline for a possible future skill, but do not build the skill yet.",
"family": "future_outline_vs_build"
},
{
"text": "Make a future-ready outline for this skill idea, but do not build the package.",
"family": "future_outline_vs_build"
}
]
}
failures/one-off-vs-reusable/cases.json
+19
View File
@@ -0,0 +1,19 @@
{
"recommended_threshold": 0.33,
"should_trigger": [],
"should_not_trigger": [],
"near_neighbor": [
{
"text": "Create a one-off prompt for this task.",
"family": "one_off_vs_reusable"
},
{
"text": "Write a custom answer for this request without creating a skill package.",
"family": "one_off_vs_reusable"
},
{
"text": "Give me a single-use prompt for today's task, not a reusable skill.",
"family": "one_off_vs_reusable"
}
]
}
scripts/run_eval_suite.py
+14 -1
View File
@@ -42,6 +42,7 @@ def main() -> None:
suites = {}
aggregate = {"false_positives": 0, "false_negatives": 0, "precision": [], "recall": []}
family_summary: dict[str, dict] = {}
for name in ("train", "dev", "holdout"):
report = run_case(script, description, baseline, root / name / "trigger_cases.json")
suites[name] = report
@@ -51,6 +52,18 @@ def main() -> None:
aggregate["precision"].append(report["precision"])
if report["recall"] is not None:
aggregate["recall"].append(report["recall"])
for family, stats in report.get("family_stats", {}).items():
slot = family_summary.setdefault(
family,
{"total": 0, "passed": 0, "false_positives": 0, "false_negatives": 0},
)
slot["total"] += stats["total"]
slot["passed"] += stats["passed"]
slot["false_positives"] += stats["false_positives"]
slot["false_negatives"] += stats["false_negatives"]
for family, stats in family_summary.items():
stats["pass_rate"] = round(stats["passed"] / stats["total"], 3) if stats["total"] else None
summary = {
"suite_count": len(suites),
@@ -59,7 +72,7 @@ def main() -> None:
"average_precision": round(sum(aggregate["precision"]) / len(aggregate["precision"]), 3) if aggregate["precision"] else None,
"average_recall": round(sum(aggregate["recall"]) / len(aggregate["recall"]), 3) if aggregate["recall"] else None,
}
output = {"summary": summary, "suites": suites}
output = {"summary": summary, "family_summary": family_summary, "suites": suites}
print(json.dumps(output, ensure_ascii=False, indent=2))
if summary["false_positives"] > 0 or summary["false_negatives"] > 0:
raise SystemExit(2)
tests/verify_failure_regressions.py
+66
View File
@@ -0,0 +1,66 @@
#!/usr/bin/env python3
import json
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parents[1]
FAILURES_DIR = ROOT / "failures"
TRIGGER_EVAL = ROOT / "scripts" / "trigger_eval.py"
DESCRIPTION = ROOT / "evals" / "improved_description.txt"
BASELINE = ROOT / "evals" / "baseline_description.txt"
SEMANTIC_CONFIG = ROOT / "evals" / "semantic_config.json"
def run_case(case_file: Path) -> dict:
proc = subprocess.run(
[
sys.executable,
str(TRIGGER_EVAL),
"--description-file",
str(DESCRIPTION),
"--baseline-description-file",
str(BASELINE),
"--cases",
str(case_file),
"--semantic-config",
str(SEMANTIC_CONFIG),
],
capture_output=True,
text=True,
)
payload = json.loads(proc.stdout)
payload["returncode"] = proc.returncode
payload["case_file"] = str(case_file.relative_to(ROOT))
return payload
def main() -> None:
case_files = sorted(FAILURES_DIR.glob("*/cases.json"))
reports = [run_case(path) for path in case_files]
failures = []
for report in reports:
if report["returncode"] != 0:
failures.append(
{
"case_file": report["case_file"],
"false_positives": report["false_positives"],
"false_negatives": report["false_negatives"],
"misfires": report["misfires"],
}
)
output = {
"ok": not failures,
"case_count": len(reports),
"families": [report["case_file"] for report in reports],
"failures": failures,
}
print(json.dumps(output, ensure_ascii=False, indent=2))
if failures:
raise SystemExit(2)
if __name__ == "__main__":
main()