feat: add examples evals and stronger packaging checks
This commit is contained in:
yaojingang
@@ -21,6 +21,14 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
2. Use `yao-meta-skill` to generate or improve the package in scaffold, production, or library mode.
|
||||
3. Run `context_sizer.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result.
|
||||
|
||||
## 5-Minute Workflow
|
||||
|
||||
1. Start from a raw workflow note.
|
||||
2. Turn it into a skill package with `SKILL.md`, `agents/interface.yaml`, and only the folders the workflow actually needs.
|
||||
3. Validate the trigger description with `evals/trigger_cases.json`.
|
||||
4. Export compatibility artifacts for the clients you care about.
|
||||
5. Compare the result against the examples in `examples/`.
|
||||
|
||||
## What It Does
|
||||
|
||||
This project helps you create, refactor, evaluate, and package skills as durable capability bundles rather than one-off prompts.
|
||||
@@ -54,6 +62,8 @@ yao-meta-skill/
|
||||
├── .gitignore
|
||||
├── agents/
|
||||
│ └── interface.yaml
|
||||
├── evals/
|
||||
├── examples/
|
||||
├── references/
|
||||
├── scripts/
|
||||
└── templates/
|
||||
@@ -77,9 +87,17 @@ Long-form material that should not bloat the main skill file. This includes desi
|
||||
|
||||
Utility scripts that make the meta-skill operational:
|
||||
|
||||
- `trigger_eval.py`: checks whether a trigger description is too broad or too weak
|
||||
- `trigger_eval.py`: evaluates trigger descriptions with positive, negative, and near-neighbor prompts
|
||||
- `context_sizer.py`: estimates context weight and warns when the initial load gets too large
|
||||
- `cross_packager.py`: builds client-specific export artifacts from the neutral source package
|
||||
- `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation
|
||||
|
||||
### `evals/`
|
||||
|
||||
Reusable trigger and packaging checks, including baseline and improved descriptions for comparison.
|
||||
|
||||
### `examples/`
|
||||
|
||||
Three end-to-end examples showing raw workflow input, design summary, and final generated skill shape.
|
||||
|
||||
### `templates/`
|
||||
|
||||
@@ -113,9 +131,9 @@ The typical flow is:
|
||||
Examples:
|
||||
|
||||
```bash
|
||||
python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --zip
|
||||
python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --expectations evals/packaging_expectations.json --zip
|
||||
python3 scripts/context_sizer.py ./yao-meta-skill
|
||||
python3 scripts/trigger_eval.py --description "Create and improve agent skills..." --cases ./cases.json
|
||||
python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
```
|
||||
|
||||
## Advantages
|
||||
@@ -145,6 +163,12 @@ This project is best for:
|
||||
| Français | [docs/README.fr-FR.md](docs/README.fr-FR.md) |
|
||||
| Русский | [docs/README.ru-RU.md](docs/README.ru-RU.md) |
|
||||
|
||||
## Examples And Evals
|
||||
|
||||
- Examples: [examples/README.md](examples/README.md)
|
||||
- Evals: [evals/README.md](evals/README.md)
|
||||
- Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md)
|
||||
|
||||
## License
|
||||
|
||||
MIT. See [LICENSE](LICENSE).
|
||||
|
||||
@@ -0,0 +1,19 @@
|
||||
# Evals
|
||||
|
||||
This directory makes trigger quality and packaging quality more reproducible.
|
||||
|
||||
Contents:
|
||||
|
||||
- `trigger_cases.json`: positive, negative, and near-neighbor prompts
|
||||
- `baseline_description.txt`: intentionally weaker trigger description
|
||||
- `improved_description.txt`: current stronger trigger description
|
||||
- `sample_trigger_report.json`: example comparison output using threshold `0.35`
|
||||
- `packaging_expectations.json`: required packaging behaviors for supported targets
|
||||
|
||||
Use:
|
||||
|
||||
```bash
|
||||
python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --threshold 0.35
|
||||
python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt --threshold 0.35
|
||||
python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip
|
||||
```
|
||||
@@ -0,0 +1 @@
|
||||
Create and improve agent skills.
|
||||
@@ -0,0 +1 @@
|
||||
Create, refactor, evaluate, and package agent skills from workflows, transcripts, prompts, docs, or rough notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, optimize skill triggering, add evals, or prepare a skill for team reuse.
|
||||
@@ -0,0 +1,7 @@
|
||||
{
|
||||
"required_targets": ["openai", "claude", "generic"],
|
||||
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
|
||||
"openai_required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
|
||||
"claude_required_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
|
||||
"generic_required_files": ["targets/generic/adapter.json"]
|
||||
}
|
||||
@@ -0,0 +1,37 @@
|
||||
{
|
||||
"threshold": 0.35,
|
||||
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.",
|
||||
"false_positives": 2,
|
||||
"false_negatives": 0,
|
||||
"precision": 0.667,
|
||||
"recall": 1.0,
|
||||
"bucket_stats": {
|
||||
"should_trigger": {
|
||||
"total": 4,
|
||||
"passed": 4,
|
||||
"pass_rate": 1.0
|
||||
},
|
||||
"should_not_trigger": {
|
||||
"total": 3,
|
||||
"passed": 3,
|
||||
"pass_rate": 1.0
|
||||
},
|
||||
"near_neighbor": {
|
||||
"total": 3,
|
||||
"passed": 1,
|
||||
"pass_rate": 0.333
|
||||
}
|
||||
},
|
||||
"comparison": {
|
||||
"baseline_false_positives": 0,
|
||||
"baseline_false_negatives": 4,
|
||||
"improved_false_positives": 2,
|
||||
"improved_false_negatives": 0,
|
||||
"false_positive_delta": 2,
|
||||
"false_negative_delta": -4,
|
||||
"baseline_precision": null,
|
||||
"improved_precision": 0.667,
|
||||
"baseline_recall": 0.0,
|
||||
"improved_recall": 1.0
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,18 @@
|
||||
{
|
||||
"should_trigger": [
|
||||
"Create a skill from this repeated workflow.",
|
||||
"Improve this skill description and add evals.",
|
||||
"Turn this runbook into a reusable agent skill.",
|
||||
"Package this skill for team reuse."
|
||||
],
|
||||
"should_not_trigger": [
|
||||
"Explain what a workflow is.",
|
||||
"Write a product headline for this landing page.",
|
||||
"Summarize this random note."
|
||||
],
|
||||
"near_neighbor": [
|
||||
"Create a one-off prompt for this task.",
|
||||
"Help me brainstorm process ideas without building a skill.",
|
||||
"Improve this README but do not turn it into a skill."
|
||||
]
|
||||
}
|
||||
@@ -0,0 +1,15 @@
|
||||
# Examples
|
||||
|
||||
This directory shows complete examples from raw workflow input to final skill package shape.
|
||||
|
||||
It is organized in three tiers:
|
||||
|
||||
- `simple-note-cleanup`: a small personal workflow
|
||||
- `team-frontend-review`: a team-level reusable workflow
|
||||
- `complex-release-orchestrator`: a multi-step, higher-complexity workflow
|
||||
|
||||
Each example contains:
|
||||
|
||||
- `raw-workflow.md`: what the user originally gives
|
||||
- `design-summary.md`: how the meta-skill scopes and structures the result
|
||||
- `generated-skill/`: the resulting skill package shape
|
||||
@@ -0,0 +1,7 @@
|
||||
# Design Summary
|
||||
|
||||
- Skill tier: complex
|
||||
- Trigger surface: release prep, release checklist, rollout coordination, rollback planning
|
||||
- Deterministic content: checklist stages and required artifacts
|
||||
- Flexible content: risk commentary and rollout recommendation
|
||||
- Package choice: `SKILL.md`, `agents/interface.yaml`, `references/`, `scripts/`, `evals/`
|
||||
@@ -0,0 +1,18 @@
|
||||
---
|
||||
name: release-orchestrator
|
||||
description: Coordinate software release preparation, rollout readiness, migration notes, rollback planning, and stakeholder communication. Use when asked to prepare a release, build a rollout checklist, review release risk, or organize launch readiness.
|
||||
---
|
||||
|
||||
# Release Orchestrator
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Gather release inputs: changes, risks, migrations, dependencies.
|
||||
2. Build artifacts for changelog, rollout, rollback, and announcements.
|
||||
3. Flag missing release-critical information before declaring readiness.
|
||||
4. Output a release package with clear go/no-go signals.
|
||||
|
||||
## Reference Map
|
||||
|
||||
- Read `references/release-checklist.md` for the stage checklist.
|
||||
- Use `scripts/build_release_packet.py` for deterministic artifact assembly.
|
||||
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "Release Orchestrator"
|
||||
short_description: "Organize release readiness, rollout, and rollback artifacts"
|
||||
default_prompt: "Use $release-orchestrator to prepare this release package and rollout checklist."
|
||||
@@ -0,0 +1,8 @@
|
||||
# Release Checklist
|
||||
|
||||
- change summary
|
||||
- dependency review
|
||||
- migration notes
|
||||
- rollout checklist
|
||||
- rollback checklist
|
||||
- stakeholder communication
|
||||
@@ -0,0 +1,2 @@
|
||||
#!/usr/bin/env python3
|
||||
print("Build release packet from gathered inputs.")
|
||||
@@ -0,0 +1,12 @@
|
||||
# Raw Workflow
|
||||
|
||||
Every release, we coordinate:
|
||||
|
||||
- changelog preparation
|
||||
- dependency diff review
|
||||
- migration notes
|
||||
- rollout checklist
|
||||
- rollback checklist
|
||||
- stakeholder announcement
|
||||
|
||||
This is currently spread across chat messages, wiki pages, and ad hoc reminders.
|
||||
@@ -0,0 +1,7 @@
|
||||
# Design Summary
|
||||
|
||||
- Skill tier: simple
|
||||
- Trigger surface: meeting notes cleanup, action item extraction, decision summary
|
||||
- Deterministic content: output sections
|
||||
- Flexible content: wording and compression style
|
||||
- Package choice: `SKILL.md` + `agents/interface.yaml`
|
||||
@@ -0,0 +1,13 @@
|
||||
---
|
||||
name: note-cleanup
|
||||
description: Clean messy meeting notes into structured markdown summaries. Use when asked to organize meeting notes, extract action items, separate decisions from open questions, or turn rough notes into a clean recap.
|
||||
---
|
||||
|
||||
# Note Cleanup
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Read the notes fully before restructuring.
|
||||
2. Remove duplicates and obvious noise.
|
||||
3. Produce sections for summary, decisions, action items, and open questions.
|
||||
4. Preserve important names, dates, and owners.
|
||||
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "Note Cleanup"
|
||||
short_description: "Turn rough meeting notes into clean recaps"
|
||||
default_prompt: "Use $note-cleanup to clean these meeting notes into a summary with decisions and action items."
|
||||
@@ -0,0 +1,14 @@
|
||||
# Raw Workflow
|
||||
|
||||
After meetings, I usually paste messy notes into chat and ask for cleanup.
|
||||
|
||||
What I actually need:
|
||||
|
||||
- remove duplicates
|
||||
- group action items
|
||||
- separate decisions from open questions
|
||||
- output a clean markdown summary
|
||||
|
||||
Typical prompt:
|
||||
|
||||
"Turn these rough meeting notes into a clean summary with action items and decisions."
|
||||
@@ -0,0 +1,7 @@
|
||||
# Design Summary
|
||||
|
||||
- Skill tier: team-level production
|
||||
- Trigger surface: React review, frontend review, accessibility review, UI security review
|
||||
- Deterministic content: review checklist and output format
|
||||
- Flexible content: severity judgment and explanation
|
||||
- Package choice: `SKILL.md`, `agents/interface.yaml`, `references/checklist.md`, `evals/`
|
||||
@@ -0,0 +1,17 @@
|
||||
---
|
||||
name: frontend-review
|
||||
description: Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality.
|
||||
---
|
||||
|
||||
# Frontend Review
|
||||
|
||||
## Workflow
|
||||
|
||||
1. Identify the UI surface and user flows affected.
|
||||
2. Check accessibility, security-sensitive inputs, loading states, and error states.
|
||||
3. Report findings by severity with concrete code references.
|
||||
4. Prefer actionable fixes over generic advice.
|
||||
|
||||
## Reference Map
|
||||
|
||||
- Read `references/checklist.md` for the review standard.
|
||||
@@ -0,0 +1,4 @@
|
||||
interface:
|
||||
display_name: "Frontend Review"
|
||||
short_description: "Review UI code for a11y, security, and UX regressions"
|
||||
default_prompt: "Use $frontend-review to review this React code before merge."
|
||||
@@ -0,0 +1,8 @@
|
||||
# Review Checklist
|
||||
|
||||
- Accessibility semantics
|
||||
- Keyboard navigation
|
||||
- Form validation and unsafe rendering risks
|
||||
- Loading states
|
||||
- Error states
|
||||
- Empty states
|
||||
@@ -0,0 +1,14 @@
|
||||
# Raw Workflow
|
||||
|
||||
Before merging frontend code, our team manually checks:
|
||||
|
||||
- accessibility issues
|
||||
- obvious security mistakes
|
||||
- risky form handling
|
||||
- missing loading and error states
|
||||
|
||||
People ask for this in many ways:
|
||||
|
||||
- "Review this React component before merge"
|
||||
- "Check this UI for a11y and security problems"
|
||||
- "Do a frontend review using our standards"
|
||||
@@ -0,0 +1,34 @@
|
||||
# Packaging Contracts
|
||||
|
||||
`cross_packager.py` is not just an export helper. It defines and validates platform contracts.
|
||||
|
||||
## Current Targets
|
||||
|
||||
- `openai`
|
||||
- `claude`
|
||||
- `generic`
|
||||
|
||||
## Contract Shape
|
||||
|
||||
Each target contract defines:
|
||||
|
||||
- required output fields
|
||||
- required output files
|
||||
- field mapping from the neutral source metadata
|
||||
|
||||
## Failure Handling
|
||||
|
||||
When `--expectations` is provided:
|
||||
|
||||
- missing required files cause exit code `2`
|
||||
- missing required fields cause exit code `2`
|
||||
- validation failures are emitted in the JSON report
|
||||
|
||||
## Source Of Truth
|
||||
|
||||
The neutral source remains:
|
||||
|
||||
- `SKILL.md`
|
||||
- `agents/interface.yaml`
|
||||
|
||||
Target-specific metadata is generated at packaging time.
|
||||
@@ -84,6 +84,37 @@ def build_manifest(skill_dir: Path, platform: str) -> dict:
|
||||
}
|
||||
|
||||
|
||||
PLATFORM_CONTRACTS = {
|
||||
"openai": {
|
||||
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
|
||||
"required_files": ["targets/openai/adapter.json", "targets/openai/agents/openai.yaml"],
|
||||
"field_mapping": {
|
||||
"display_name": "interface.display_name",
|
||||
"short_description": "interface.short_description",
|
||||
"default_prompt": "interface.default_prompt",
|
||||
},
|
||||
},
|
||||
"claude": {
|
||||
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
|
||||
"required_files": ["targets/claude/adapter.json", "targets/claude/README.md"],
|
||||
"field_mapping": {
|
||||
"display_name": "adapter.display_name",
|
||||
"short_description": "adapter.short_description",
|
||||
"default_prompt": "adapter.default_prompt",
|
||||
},
|
||||
},
|
||||
"generic": {
|
||||
"required_fields": ["name", "description", "version", "display_name", "short_description", "default_prompt", "canonical_metadata"],
|
||||
"required_files": ["targets/generic/adapter.json"],
|
||||
"field_mapping": {
|
||||
"display_name": "adapter.display_name",
|
||||
"short_description": "adapter.short_description",
|
||||
"default_prompt": "adapter.default_prompt",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def write_yaml_like(path: Path, payload: dict) -> None:
|
||||
interface = payload.get("interface", {})
|
||||
lines = ["interface:"]
|
||||
@@ -121,6 +152,7 @@ def write_adapter(skill_dir: Path, out_dir: Path, platform: str) -> Path:
|
||||
else:
|
||||
payload["install_hint"] = f"Use {skill_dir.name} as an Agent Skills compatible package."
|
||||
path = target_dir / "adapter.json"
|
||||
payload["contract"] = PLATFORM_CONTRACTS.get(platform, PLATFORM_CONTRACTS["generic"])
|
||||
path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8")
|
||||
return path
|
||||
|
||||
@@ -143,11 +175,43 @@ def copy_manifest(skill_dir: Path, out_dir: Path) -> Path:
|
||||
return manifest_path
|
||||
|
||||
|
||||
def load_expectations(path: Path | None) -> dict:
|
||||
if path is None:
|
||||
return {}
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def validate_exports(out_dir: Path, expectations: dict) -> dict:
|
||||
failures = []
|
||||
required_targets = expectations.get("required_targets", [])
|
||||
required_fields = expectations.get("required_fields", [])
|
||||
required_by_target = {
|
||||
"openai": expectations.get("openai_required_files", []),
|
||||
"claude": expectations.get("claude_required_files", []),
|
||||
"generic": expectations.get("generic_required_files", []),
|
||||
}
|
||||
|
||||
for target in required_targets:
|
||||
adapter_path = out_dir / "targets" / target / "adapter.json"
|
||||
if not adapter_path.exists():
|
||||
failures.append(f"missing adapter for target: {target}")
|
||||
continue
|
||||
payload = json.loads(adapter_path.read_text(encoding="utf-8"))
|
||||
for field in required_fields:
|
||||
if field not in payload:
|
||||
failures.append(f"missing field '{field}' in {adapter_path.relative_to(out_dir)}")
|
||||
for rel in required_by_target.get(target, []):
|
||||
if not (out_dir / rel).exists():
|
||||
failures.append(f"missing file: {rel}")
|
||||
return {"ok": not failures, "failures": failures}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Generate lightweight cross-platform packaging artifacts.")
|
||||
parser.add_argument("skill_dir", help="Path to the skill directory")
|
||||
parser.add_argument("--platform", action="append", default=[], help="Target platform: openai, claude, generic")
|
||||
parser.add_argument("--output-dir", default="dist", help="Output directory")
|
||||
parser.add_argument("--expectations", help="JSON file describing packaging expectations")
|
||||
parser.add_argument("--zip", action="store_true", help="Create a zip package")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -164,7 +228,21 @@ def main() -> None:
|
||||
if args.zip:
|
||||
generated.append(str(make_zip(skill_dir, out_dir)))
|
||||
|
||||
print(json.dumps({"output_dir": str(out_dir), "generated": generated}, ensure_ascii=False, indent=2))
|
||||
expectations = load_expectations(Path(args.expectations).resolve()) if args.expectations else {}
|
||||
validation = validate_exports(out_dir, expectations) if expectations else None
|
||||
report = {
|
||||
"output_dir": str(out_dir),
|
||||
"generated": generated,
|
||||
"contracts": PLATFORM_CONTRACTS,
|
||||
"validation": validation,
|
||||
"failure_handling": {
|
||||
"missing_required_file": "exit with code 2 when expectations are provided and validation fails",
|
||||
"missing_required_field": "exit with code 2 when expectations are provided and validation fails",
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if validation and not validation["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
+73
-12
@@ -37,43 +37,96 @@ def score_prompt(description_words: set[str], prompt: str) -> float:
|
||||
return len(overlap) / len(prompt_words)
|
||||
|
||||
|
||||
def classify_bucket(bucket: str) -> bool:
|
||||
return bucket == "should_trigger"
|
||||
|
||||
|
||||
def evaluate(description: str, cases: dict, threshold: float) -> dict:
|
||||
desc_words = words(description)
|
||||
results = {"should_trigger": [], "should_not_trigger": []}
|
||||
results = {"should_trigger": [], "should_not_trigger": [], "near_neighbor": []}
|
||||
fp = 0
|
||||
fn = 0
|
||||
bucket_stats = {}
|
||||
misfires = []
|
||||
|
||||
for bucket in ("should_trigger", "should_not_trigger"):
|
||||
for bucket in ("should_trigger", "should_not_trigger", "near_neighbor"):
|
||||
expected = classify_bucket(bucket)
|
||||
total = 0
|
||||
passed_count = 0
|
||||
for prompt in cases.get(bucket, []):
|
||||
score = score_prompt(desc_words, prompt)
|
||||
predicted = score >= threshold
|
||||
expected = bucket == "should_trigger"
|
||||
passed = predicted == expected
|
||||
total += 1
|
||||
if passed:
|
||||
passed_count += 1
|
||||
if not passed and expected:
|
||||
fn += 1
|
||||
if not passed and not expected:
|
||||
fp += 1
|
||||
results[bucket].append(
|
||||
{
|
||||
"prompt": prompt,
|
||||
"score": round(score, 3),
|
||||
"predicted_trigger": predicted,
|
||||
"passed": passed,
|
||||
}
|
||||
)
|
||||
record = {
|
||||
"prompt": prompt,
|
||||
"score": round(score, 3),
|
||||
"predicted_trigger": predicted,
|
||||
"expected_trigger": expected,
|
||||
"passed": passed,
|
||||
}
|
||||
if 0.75 * threshold <= score <= 1.25 * threshold:
|
||||
record["boundary_case"] = True
|
||||
results[bucket].append(record)
|
||||
if not passed:
|
||||
misfires.append(
|
||||
{
|
||||
"bucket": bucket,
|
||||
"prompt": prompt,
|
||||
"score": round(score, 3),
|
||||
"reason": "false_negative" if expected else "false_positive",
|
||||
}
|
||||
)
|
||||
bucket_stats[bucket] = {
|
||||
"total": total,
|
||||
"passed": passed_count,
|
||||
"pass_rate": round(passed_count / total, 3) if total else None,
|
||||
}
|
||||
|
||||
tp = sum(1 for item in results["should_trigger"] if item["predicted_trigger"])
|
||||
precision = tp / (tp + fp) if (tp + fp) else None
|
||||
recall = tp / (tp + fn) if (tp + fn) else None
|
||||
|
||||
return {
|
||||
"threshold": threshold,
|
||||
"threshold_explanation": "Prompts at or above the threshold are treated as trigger matches. Scores near the threshold should be reviewed as boundary cases.",
|
||||
"false_positives": fp,
|
||||
"false_negatives": fn,
|
||||
"precision": round(precision, 3) if precision is not None else None,
|
||||
"recall": round(recall, 3) if recall is not None else None,
|
||||
"bucket_stats": bucket_stats,
|
||||
"misfires": misfires,
|
||||
"results": results,
|
||||
}
|
||||
|
||||
|
||||
def compare_reports(baseline: dict, improved: dict) -> dict:
|
||||
return {
|
||||
"baseline_false_positives": baseline["false_positives"],
|
||||
"baseline_false_negatives": baseline["false_negatives"],
|
||||
"improved_false_positives": improved["false_positives"],
|
||||
"improved_false_negatives": improved["false_negatives"],
|
||||
"false_positive_delta": improved["false_positives"] - baseline["false_positives"],
|
||||
"false_negative_delta": improved["false_negatives"] - baseline["false_negatives"],
|
||||
"baseline_precision": baseline["precision"],
|
||||
"improved_precision": improved["precision"],
|
||||
"baseline_recall": baseline["recall"],
|
||||
"improved_recall": improved["recall"],
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Heuristic trigger quality evaluator.")
|
||||
parser.add_argument("--description", help="Description string to evaluate")
|
||||
parser.add_argument("--description-file", help="Read description text from file")
|
||||
parser.add_argument("--baseline-description", help="Baseline description string to compare against")
|
||||
parser.add_argument("--baseline-description-file", help="Read baseline description from file")
|
||||
parser.add_argument("--cases", required=True, help="JSON file with should_trigger and should_not_trigger arrays")
|
||||
parser.add_argument("--threshold", type=float, default=0.18, help="Token overlap threshold")
|
||||
args = parser.parse_args()
|
||||
@@ -84,7 +137,15 @@ def main() -> None:
|
||||
if not description:
|
||||
raise SystemExit("Provide --description or --description-file")
|
||||
|
||||
report = evaluate(description, load_cases(Path(args.cases)), args.threshold)
|
||||
cases = load_cases(Path(args.cases))
|
||||
report = evaluate(description, cases, args.threshold)
|
||||
|
||||
baseline = args.baseline_description
|
||||
if args.baseline_description_file:
|
||||
baseline = extract_description(Path(args.baseline_description_file).read_text(encoding="utf-8"))
|
||||
if baseline:
|
||||
report["comparison"] = compare_reports(evaluate(baseline, cases, args.threshold), report)
|
||||
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if report["false_positives"] > 2:
|
||||
raise SystemExit(2)
|
||||
|
||||
Reference in New Issue
Block a user