diff --git a/README.md b/README.md index e480742..fbfa48c 100644 --- a/README.md +++ b/README.md @@ -21,6 +21,14 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl 2. Use `yao-meta-skill` to generate or improve the package in scaffold, production, or library mode. 3. Run `context_sizer.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result. +## 5-Minute Workflow + +1. Start from a raw workflow note. +2. Turn it into a skill package with `SKILL.md`, `agents/interface.yaml`, and only the folders the workflow actually needs. +3. Validate the trigger description with `evals/trigger_cases.json`. +4. Export compatibility artifacts for the clients you care about. +5. Compare the result against the examples in `examples/`. + ## What It Does This project helps you create, refactor, evaluate, and package skills as durable capability bundles rather than one-off prompts. @@ -54,6 +62,8 @@ yao-meta-skill/ ├── .gitignore ├── agents/ │ └── interface.yaml +├── evals/ +├── examples/ ├── references/ ├── scripts/ └── templates/ @@ -77,9 +87,17 @@ Long-form material that should not bloat the main skill file. This includes desi Utility scripts that make the meta-skill operational: -- `trigger_eval.py`: checks whether a trigger description is too broad or too weak +- `trigger_eval.py`: evaluates trigger descriptions with positive, negative, and near-neighbor prompts - `context_sizer.py`: estimates context weight and warns when the initial load gets too large -- `cross_packager.py`: builds client-specific export artifacts from the neutral source package +- `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation + +### `evals/` + +Reusable trigger and packaging checks, including baseline and improved descriptions for comparison. + +### `examples/` + +Three end-to-end examples showing raw workflow input, design summary, and final generated skill shape. ### `templates/` @@ -113,9 +131,9 @@ The typical flow is: Examples: ```bash -python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --zip +python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --expectations evals/packaging_expectations.json --zip python3 scripts/context_sizer.py ./yao-meta-skill -python3 scripts/trigger_eval.py --description "Create and improve agent skills..." --cases ./cases.json +python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt ``` ## Advantages @@ -145,6 +163,12 @@ This project is best for: | Français | [docs/README.fr-FR.md](docs/README.fr-FR.md) | | Русский | [docs/README.ru-RU.md](docs/README.ru-RU.md) | +## Examples And Evals + +- Examples: [examples/README.md](examples/README.md) +- Evals: [evals/README.md](evals/README.md) +- Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md) + ## License MIT. See [LICENSE](LICENSE). diff --git a/evals/README.md b/evals/README.md new file mode 100644 index 0000000..64d82bf --- /dev/null +++ b/evals/README.md @@ -0,0 +1,19 @@ +# Evals + +This directory makes trigger quality and packaging quality more reproducible. + +Contents: + +- `trigger_cases.json`: positive, negative, and near-neighbor prompts +- `baseline_description.txt`: intentionally weaker trigger description +- `improved_description.txt`: current stronger trigger description +- `sample_trigger_report.json`: example comparison output using threshold `0.35` +- `packaging_expectations.json`: required packaging behaviors for supported targets + +Use: + +```bash +python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --threshold 0.35 +python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt --threshold 0.35 +python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip +``` diff --git a/evals/baseline_description.txt b/evals/baseline_description.txt new file mode 100644 index 0000000..b8f15b2 --- /dev/null +++ b/evals/baseline_description.txt @@ -0,0 +1 @@ +Create and improve agent skills. diff --git a/evals/improved_description.txt b/evals/improved_description.txt new file mode 100644 index 0000000..c94861c --- /dev/null +++ b/evals/improved_description.txt @@ -0,0 +1 @@ +Create, refactor, evaluate, and package agent skills from workflows, transcripts, prompts, docs, or rough notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, optimize skill triggering, add evals, or prepare a skill for team reuse. diff --git a/evals/packaging_expectations.json b/evals/packaging_expectations.json new file mode 100644 index 0000000..b93ee23 --- /dev/null +++ b/evals/packaging_expectations.json @@ -0,0 +1,7 @@ +{ + "required_targets": ["openai", "claude", "generic"], + "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "openai_required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], + "claude_required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], + "generic_required_files": ["targets/generic/adapter.json"] +} diff --git a/evals/sample_trigger_report.json b/evals/sample_trigger_report.json new file mode 100644 index 0000000..48dc508 --- /dev/null +++ b/evals/sample_trigger_report.json @@ -0,0 +1,37 @@ +{ + "threshold": 0.35, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.", + "false_positives": 2, + "false_negatives": 0, + "precision": 0.667, + "recall": 1.0, + "bucket_stats": { + "should_trigger": { + "total": 4, + "passed": 4, + "pass_rate": 1.0 + }, + "should_not_trigger": { + "total": 3, + "passed": 3, + "pass_rate": 1.0 + }, + "near_neighbor": { + "total": 3, + "passed": 1, + "pass_rate": 0.333 + } + }, + "comparison": { + "baseline_false_positives": 0, + "baseline_false_negatives": 4, + "improved_false_positives": 2, + "improved_false_negatives": 0, + "false_positive_delta": 2, + "false_negative_delta": -4, + "baseline_precision": null, + "improved_precision": 0.667, + "baseline_recall": 0.0, + "improved_recall": 1.0 + } +} diff --git a/evals/trigger_cases.json b/evals/trigger_cases.json new file mode 100644 index 0000000..9f72f06 --- /dev/null +++ b/evals/trigger_cases.json @@ -0,0 +1,18 @@ +{ + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Improve this skill description and add evals.", + "Turn this runbook into a reusable agent skill.", + "Package this skill for team reuse." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Write a product headline for this landing page.", + "Summarize this random note." + ], + "near_neighbor": [ + "Create a one-off prompt for this task.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill." + ] +} diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 0000000..ddcb77e --- /dev/null +++ b/examples/README.md @@ -0,0 +1,15 @@ +# Examples + +This directory shows complete examples from raw workflow input to final skill package shape. + +It is organized in three tiers: + +- `simple-note-cleanup`: a small personal workflow +- `team-frontend-review`: a team-level reusable workflow +- `complex-release-orchestrator`: a multi-step, higher-complexity workflow + +Each example contains: + +- `raw-workflow.md`: what the user originally gives +- `design-summary.md`: how the meta-skill scopes and structures the result +- `generated-skill/`: the resulting skill package shape diff --git a/examples/complex-release-orchestrator/design-summary.md b/examples/complex-release-orchestrator/design-summary.md new file mode 100644 index 0000000..b436af8 --- /dev/null +++ b/examples/complex-release-orchestrator/design-summary.md @@ -0,0 +1,7 @@ +# Design Summary + +- Skill tier: complex +- Trigger surface: release prep, release checklist, rollout coordination, rollback planning +- Deterministic content: checklist stages and required artifacts +- Flexible content: risk commentary and rollout recommendation +- Package choice: `SKILL.md`, `agents/interface.yaml`, `references/`, `scripts/`, `evals/` diff --git a/examples/complex-release-orchestrator/generated-skill/SKILL.md b/examples/complex-release-orchestrator/generated-skill/SKILL.md new file mode 100644 index 0000000..d57a7b8 --- /dev/null +++ b/examples/complex-release-orchestrator/generated-skill/SKILL.md @@ -0,0 +1,18 @@ +--- +name: release-orchestrator +description: Coordinate software release preparation, rollout readiness, migration notes, rollback planning, and stakeholder communication. Use when asked to prepare a release, build a rollout checklist, review release risk, or organize launch readiness. +--- + +# Release Orchestrator + +## Workflow + +1. Gather release inputs: changes, risks, migrations, dependencies. +2. Build artifacts for changelog, rollout, rollback, and announcements. +3. Flag missing release-critical information before declaring readiness. +4. Output a release package with clear go/no-go signals. + +## Reference Map + +- Read `references/release-checklist.md` for the stage checklist. +- Use `scripts/build_release_packet.py` for deterministic artifact assembly. diff --git a/examples/complex-release-orchestrator/generated-skill/agents/interface.yaml b/examples/complex-release-orchestrator/generated-skill/agents/interface.yaml new file mode 100644 index 0000000..baa1410 --- /dev/null +++ b/examples/complex-release-orchestrator/generated-skill/agents/interface.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Release Orchestrator" + short_description: "Organize release readiness, rollout, and rollback artifacts" + default_prompt: "Use $release-orchestrator to prepare this release package and rollout checklist." diff --git a/examples/complex-release-orchestrator/generated-skill/references/release-checklist.md b/examples/complex-release-orchestrator/generated-skill/references/release-checklist.md new file mode 100644 index 0000000..db62b49 --- /dev/null +++ b/examples/complex-release-orchestrator/generated-skill/references/release-checklist.md @@ -0,0 +1,8 @@ +# Release Checklist + +- change summary +- dependency review +- migration notes +- rollout checklist +- rollback checklist +- stakeholder communication diff --git a/examples/complex-release-orchestrator/generated-skill/scripts/build_release_packet.py b/examples/complex-release-orchestrator/generated-skill/scripts/build_release_packet.py new file mode 100644 index 0000000..179c287 --- /dev/null +++ b/examples/complex-release-orchestrator/generated-skill/scripts/build_release_packet.py @@ -0,0 +1,2 @@ +#!/usr/bin/env python3 +print("Build release packet from gathered inputs.") diff --git a/examples/complex-release-orchestrator/raw-workflow.md b/examples/complex-release-orchestrator/raw-workflow.md new file mode 100644 index 0000000..e5dd69d --- /dev/null +++ b/examples/complex-release-orchestrator/raw-workflow.md @@ -0,0 +1,12 @@ +# Raw Workflow + +Every release, we coordinate: + +- changelog preparation +- dependency diff review +- migration notes +- rollout checklist +- rollback checklist +- stakeholder announcement + +This is currently spread across chat messages, wiki pages, and ad hoc reminders. diff --git a/examples/simple-note-cleanup/design-summary.md b/examples/simple-note-cleanup/design-summary.md new file mode 100644 index 0000000..d48d5b2 --- /dev/null +++ b/examples/simple-note-cleanup/design-summary.md @@ -0,0 +1,7 @@ +# Design Summary + +- Skill tier: simple +- Trigger surface: meeting notes cleanup, action item extraction, decision summary +- Deterministic content: output sections +- Flexible content: wording and compression style +- Package choice: `SKILL.md` + `agents/interface.yaml` diff --git a/examples/simple-note-cleanup/generated-skill/SKILL.md b/examples/simple-note-cleanup/generated-skill/SKILL.md new file mode 100644 index 0000000..956b302 --- /dev/null +++ b/examples/simple-note-cleanup/generated-skill/SKILL.md @@ -0,0 +1,13 @@ +--- +name: note-cleanup +description: Clean messy meeting notes into structured markdown summaries. Use when asked to organize meeting notes, extract action items, separate decisions from open questions, or turn rough notes into a clean recap. +--- + +# Note Cleanup + +## Workflow + +1. Read the notes fully before restructuring. +2. Remove duplicates and obvious noise. +3. Produce sections for summary, decisions, action items, and open questions. +4. Preserve important names, dates, and owners. diff --git a/examples/simple-note-cleanup/generated-skill/agents/interface.yaml b/examples/simple-note-cleanup/generated-skill/agents/interface.yaml new file mode 100644 index 0000000..d810b95 --- /dev/null +++ b/examples/simple-note-cleanup/generated-skill/agents/interface.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Note Cleanup" + short_description: "Turn rough meeting notes into clean recaps" + default_prompt: "Use $note-cleanup to clean these meeting notes into a summary with decisions and action items." diff --git a/examples/simple-note-cleanup/raw-workflow.md b/examples/simple-note-cleanup/raw-workflow.md new file mode 100644 index 0000000..dd599bf --- /dev/null +++ b/examples/simple-note-cleanup/raw-workflow.md @@ -0,0 +1,14 @@ +# Raw Workflow + +After meetings, I usually paste messy notes into chat and ask for cleanup. + +What I actually need: + +- remove duplicates +- group action items +- separate decisions from open questions +- output a clean markdown summary + +Typical prompt: + +"Turn these rough meeting notes into a clean summary with action items and decisions." diff --git a/examples/team-frontend-review/design-summary.md b/examples/team-frontend-review/design-summary.md new file mode 100644 index 0000000..09b2094 --- /dev/null +++ b/examples/team-frontend-review/design-summary.md @@ -0,0 +1,7 @@ +# Design Summary + +- Skill tier: team-level production +- Trigger surface: React review, frontend review, accessibility review, UI security review +- Deterministic content: review checklist and output format +- Flexible content: severity judgment and explanation +- Package choice: `SKILL.md`, `agents/interface.yaml`, `references/checklist.md`, `evals/` diff --git a/examples/team-frontend-review/generated-skill/SKILL.md b/examples/team-frontend-review/generated-skill/SKILL.md new file mode 100644 index 0000000..19a7e89 --- /dev/null +++ b/examples/team-frontend-review/generated-skill/SKILL.md @@ -0,0 +1,17 @@ +--- +name: frontend-review +description: Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality. +--- + +# Frontend Review + +## Workflow + +1. Identify the UI surface and user flows affected. +2. Check accessibility, security-sensitive inputs, loading states, and error states. +3. Report findings by severity with concrete code references. +4. Prefer actionable fixes over generic advice. + +## Reference Map + +- Read `references/checklist.md` for the review standard. diff --git a/examples/team-frontend-review/generated-skill/agents/interface.yaml b/examples/team-frontend-review/generated-skill/agents/interface.yaml new file mode 100644 index 0000000..1b911d6 --- /dev/null +++ b/examples/team-frontend-review/generated-skill/agents/interface.yaml @@ -0,0 +1,4 @@ +interface: + display_name: "Frontend Review" + short_description: "Review UI code for a11y, security, and UX regressions" + default_prompt: "Use $frontend-review to review this React code before merge." diff --git a/examples/team-frontend-review/generated-skill/references/checklist.md b/examples/team-frontend-review/generated-skill/references/checklist.md new file mode 100644 index 0000000..83d0b33 --- /dev/null +++ b/examples/team-frontend-review/generated-skill/references/checklist.md @@ -0,0 +1,8 @@ +# Review Checklist + +- Accessibility semantics +- Keyboard navigation +- Form validation and unsafe rendering risks +- Loading states +- Error states +- Empty states diff --git a/examples/team-frontend-review/raw-workflow.md b/examples/team-frontend-review/raw-workflow.md new file mode 100644 index 0000000..eec92a4 --- /dev/null +++ b/examples/team-frontend-review/raw-workflow.md @@ -0,0 +1,14 @@ +# Raw Workflow + +Before merging frontend code, our team manually checks: + +- accessibility issues +- obvious security mistakes +- risky form handling +- missing loading and error states + +People ask for this in many ways: + +- "Review this React component before merge" +- "Check this UI for a11y and security problems" +- "Do a frontend review using our standards" diff --git a/references/packaging-contracts.md b/references/packaging-contracts.md new file mode 100644 index 0000000..7cc2978 --- /dev/null +++ b/references/packaging-contracts.md @@ -0,0 +1,34 @@ +# Packaging Contracts + +`cross_packager.py` is not just an export helper. It defines and validates platform contracts. + +## Current Targets + +- `openai` +- `claude` +- `generic` + +## Contract Shape + +Each target contract defines: + +- required output fields +- required output files +- field mapping from the neutral source metadata + +## Failure Handling + +When `--expectations` is provided: + +- missing required files cause exit code `2` +- missing required fields cause exit code `2` +- validation failures are emitted in the JSON report + +## Source Of Truth + +The neutral source remains: + +- `SKILL.md` +- `agents/interface.yaml` + +Target-specific metadata is generated at packaging time. diff --git a/scripts/cross_packager.py b/scripts/cross_packager.py index af3396f..29ae7fc 100644 --- a/scripts/cross_packager.py +++ b/scripts/cross_packager.py @@ -84,6 +84,37 @@ def build_manifest(skill_dir: Path, platform: str) -> dict: } +PLATFORM_CONTRACTS = { + "openai": { + "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"], + "field_mapping": { + "display_name": "interface.display_name", + "short_description": "interface.short_description", + "default_prompt": "interface.default_prompt", + }, + }, + "claude": { + "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "required_files": ["targets/claude/adapter.json", "targets/claude/README.md"], + "field_mapping": { + "display_name": "adapter.display_name", + "short_description": "adapter.short_description", + "default_prompt": "adapter.default_prompt", + }, + }, + "generic": { + "required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"], + "required_files": ["targets/generic/adapter.json"], + "field_mapping": { + "display_name": "adapter.display_name", + "short_description": "adapter.short_description", + "default_prompt": "adapter.default_prompt", + }, + }, +} + + def write_yaml_like(path: Path, payload: dict) -> None: interface = payload.get("interface", {}) lines = ["interface:"] @@ -121,6 +152,7 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path: else: payload["install_hint"] = f"Use {skill_dir.name} as an Agent Skills compatible package." path = target_dir / "adapter.json" + payload["contract"] = PLATFORM_CONTRACTS.get(platform, PLATFORM_CONTRACTS["generic"]) path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") return path @@ -143,11 +175,43 @@ def copy_manifest(skill_dir: Path, out_dir: Path) -> Path: return manifest_path +def load_expectations(path: Path | None) -> dict: + if path is None: + return {} + return json.loads(path.read_text(encoding="utf-8")) + + +def validate_exports(out_dir: Path, expectations: dict) -> dict: + failures = [] + required_targets = expectations.get("required_targets", []) + required_fields = expectations.get("required_fields", []) + required_by_target = { + "openai": expectations.get("openai_required_files", []), + "claude": expectations.get("claude_required_files", []), + "generic": expectations.get("generic_required_files", []), + } + + for target in required_targets: + adapter_path = out_dir / "targets" / target / "adapter.json" + if not adapter_path.exists(): + failures.append(f"missing adapter for target: {target}") + continue + payload = json.loads(adapter_path.read_text(encoding="utf-8")) + for field in required_fields: + if field not in payload: + failures.append(f"missing field '{field}' in {adapter_path.relative_to(out_dir)}") + for rel in required_by_target.get(target, []): + if not (out_dir / rel).exists(): + failures.append(f"missing file: {rel}") + return {"ok": not failures, "failures": failures} + + def main() -> None: parser = argparse.ArgumentParser(description="Generate lightweight cross-platform packaging artifacts.") parser.add_argument("skill_dir", help="Path to the skill directory") parser.add_argument("--platform", action="append", default=[], help="Target platform: openai, claude, generic") parser.add_argument("--output-dir", default="dist", help="Output directory") + parser.add_argument("--expectations", help="JSON file describing packaging expectations") parser.add_argument("--zip", action="store_true", help="Create a zip package") args = parser.parse_args() @@ -164,7 +228,21 @@ def main() -> None: if args.zip: generated.append(str(make_zip(skill_dir, out_dir))) - print(json.dumps({"output_dir": str(out_dir), "generated": generated}, ensure_ascii=False, indent=2)) + expectations = load_expectations(Path(args.expectations).resolve()) if args.expectations else {} + validation = validate_exports(out_dir, expectations) if expectations else None + report = { + "output_dir": str(out_dir), + "generated": generated, + "contracts": PLATFORM_CONTRACTS, + "validation": validation, + "failure_handling": { + "missing_required_file": "exit with code 2 when expectations are provided and validation fails", + "missing_required_field": "exit with code 2 when expectations are provided and validation fails", + }, + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + if validation and not validation["ok"]: + raise SystemExit(2) if __name__ == "__main__": diff --git a/scripts/trigger_eval.py b/scripts/trigger_eval.py index cf2e0c2..74ce8c5 100644 --- a/scripts/trigger_eval.py +++ b/scripts/trigger_eval.py @@ -37,43 +37,96 @@ def score_prompt(description_words: set[str], prompt: str) -> float: return len(overlap) / len(prompt_words) +def classify_bucket(bucket: str) -> bool: + return bucket == "should_trigger" + + def evaluate(description: str, cases: dict, threshold: float) -> dict: desc_words = words(description) - results = {"should_trigger": [], "should_not_trigger": []} + results = {"should_trigger": [], "should_not_trigger": [], "near_neighbor": []} fp = 0 fn = 0 + bucket_stats = {} + misfires = [] - for bucket in ("should_trigger", "should_not_trigger"): + for bucket in ("should_trigger", "should_not_trigger", "near_neighbor"): + expected = classify_bucket(bucket) + total = 0 + passed_count = 0 for prompt in cases.get(bucket, []): score = score_prompt(desc_words, prompt) predicted = score >= threshold - expected = bucket == "should_trigger" passed = predicted == expected + total += 1 + if passed: + passed_count += 1 if not passed and expected: fn += 1 if not passed and not expected: fp += 1 - results[bucket].append( - { - "prompt": prompt, - "score": round(score, 3), - "predicted_trigger": predicted, - "passed": passed, - } - ) + record = { + "prompt": prompt, + "score": round(score, 3), + "predicted_trigger": predicted, + "expected_trigger": expected, + "passed": passed, + } + if 0.75 * threshold <= score <= 1.25 * threshold: + record["boundary_case"] = True + results[bucket].append(record) + if not passed: + misfires.append( + { + "bucket": bucket, + "prompt": prompt, + "score": round(score, 3), + "reason": "false_negative" if expected else "false_positive", + } + ) + bucket_stats[bucket] = { + "total": total, + "passed": passed_count, + "pass_rate": round(passed_count / total, 3) if total else None, + } + + tp = sum(1 for item in results["should_trigger"] if item["predicted_trigger"]) + precision = tp / (tp + fp) if (tp + fp) else None + recall = tp / (tp + fn) if (tp + fn) else None return { "threshold": threshold, + "threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.", "false_positives": fp, "false_negatives": fn, + "precision": round(precision, 3) if precision is not None else None, + "recall": round(recall, 3) if recall is not None else None, + "bucket_stats": bucket_stats, + "misfires": misfires, "results": results, } +def compare_reports(baseline: dict, improved: dict) -> dict: + return { + "baseline_false_positives": baseline["false_positives"], + "baseline_false_negatives": baseline["false_negatives"], + "improved_false_positives": improved["false_positives"], + "improved_false_negatives": improved["false_negatives"], + "false_positive_delta": improved["false_positives"] - baseline["false_positives"], + "false_negative_delta": improved["false_negatives"] - baseline["false_negatives"], + "baseline_precision": baseline["precision"], + "improved_precision": improved["precision"], + "baseline_recall": baseline["recall"], + "improved_recall": improved["recall"], + } + + def main() -> None: parser = argparse.ArgumentParser(description="Heuristic trigger quality evaluator.") parser.add_argument("--description", help="Description string to evaluate") parser.add_argument("--description-file", help="Read description text from file") + parser.add_argument("--baseline-description", help="Baseline description string to compare against") + parser.add_argument("--baseline-description-file", help="Read baseline description from file") parser.add_argument("--cases", required=True, help="JSON file with should_trigger and should_not_trigger arrays") parser.add_argument("--threshold", type=float, default=0.18, help="Token overlap threshold") args = parser.parse_args() @@ -84,7 +137,15 @@ def main() -> None: if not description: raise SystemExit("Provide --description or --description-file") - report = evaluate(description, load_cases(Path(args.cases)), args.threshold) + cases = load_cases(Path(args.cases)) + report = evaluate(description, cases, args.threshold) + + baseline = args.baseline_description + if args.baseline_description_file: + baseline = extract_description(Path(args.baseline_description_file).read_text(encoding="utf-8")) + if baseline: + report["comparison"] = compare_reports(evaluate(baseline, cases, args.threshold), report) + print(json.dumps(report, ensure_ascii=False, indent=2)) if report["false_positives"] > 2: raise SystemExit(2)