diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml new file mode 100644 index 00000000..cd65d486 --- /dev/null +++ b/.github/workflows/test.yml @@ -0,0 +1,16 @@ +name: test + +on: + push: + pull_request: + +jobs: + test: + runs-on: ubuntu-latest + steps: + - uses: actions/checkout@v4 + - uses: actions/setup-python@v5 + with: + python-version: "3.11" + - run: python -m pip install --upgrade pip pyyaml + - run: make test diff --git a/Makefile b/Makefile index 1c342811..3fc9e88b 100644 --- a/Makefile +++ b/Makefile @@ -1,17 +1,26 @@ PYTHON ?= python3 -.PHONY: eval package-check package-failure-check test clean +.PHONY: eval eval-suite package-check package-failure-check validate lint test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt +eval-suite: + $(PYTHON) scripts/run_eval_suite.py + package-check: $(PYTHON) scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --output-dir dist --zip package-failure-check: $(PYTHON) tests/verify_packager_failures.py -test: eval package-check package-failure-check +validate: + $(PYTHON) scripts/validate_skill.py . + +lint: + $(PYTHON) scripts/lint_skill.py . + +test: eval eval-suite package-check package-failure-check validate lint clean: rm -rf dist tests/tmp diff --git a/README.md b/README.md index e0991217..edc8980f 100644 --- a/README.md +++ b/README.md @@ -103,8 +103,10 @@ Long-form material that should not bloat the main skill file. This includes desi Utility scripts that make the meta-skill operational: - `trigger_eval.py`: evaluates trigger descriptions with positive, negative, and near-neighbor prompts +- `run_eval_suite.py`: runs train/dev/holdout trigger suites and fails if aggregate regressions appear - `context_sizer.py`: estimates context weight and warns when the initial load gets too large - `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation +- `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain ### `evals/` @@ -114,10 +116,15 @@ Reusable trigger and packaging checks, including baseline and improved descripti Three end-to-end examples showing raw workflow input, design summary, and final generated skill shape. +### `.github/workflows/test.yml` + +Continuous integration entrypoint that runs the full local regression suite on push and pull request. + ## Validation Notes - Trigger evaluation is stronger than the original overlap-only version, but it is still heuristic. - The sample trigger report now covers a larger positive, negative, and near-neighbor set rather than a tiny demo set. +- Train/dev/holdout trigger suites now separate iterative tuning from final verification. - Packaging validation now uses explicit contracts and YAML parsing, but it is still a lightweight local validation layer rather than a full platform integration suite. - `evals/failure-cases.md` captures known weak spots that should remain part of regression checks. - `tests/verify_packager_failures.py` checks that invalid metadata, invalid YAML, and unsupported targets fail clearly. @@ -192,6 +199,7 @@ This project is best for: - Evals: [evals/README.md](evals/README.md) - Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md) - Failure fixtures: [tests/fixtures](tests/fixtures) +- Evolution example: [examples/evolution-frontend-review/README.md](examples/evolution-frontend-review/README.md) ## License diff --git a/evals/README.md b/evals/README.md index 553f6f7b..7462c910 100644 --- a/evals/README.md +++ b/evals/README.md @@ -5,6 +5,7 @@ This directory makes trigger quality and packaging quality more reproducible. Contents: - `trigger_cases.json`: positive, negative, and near-neighbor prompts +- `train/`, `dev/`, `holdout/`: split trigger suites for iterative tuning and final verification - `baseline_description.txt`: intentionally weaker trigger description - `improved_description.txt`: current stronger trigger description - `sample_trigger_report.json`: example comparison output using threshold `0.35` @@ -16,6 +17,7 @@ Use: ```bash python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --threshold 0.35 python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt --threshold 0.35 +python3 scripts/run_eval_suite.py python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip python3 tests/verify_packager_failures.py ``` @@ -28,3 +30,4 @@ Regression scope now includes: - long-context positives - mixed-intent negatives - explicit "do not build a skill" negatives +- holdout verification diff --git a/evals/dev/trigger_cases.json b/evals/dev/trigger_cases.json new file mode 100644 index 00000000..ecacefb4 --- /dev/null +++ b/evals/dev/trigger_cases.json @@ -0,0 +1,27 @@ +{ + "recommended_threshold": 0.37, + "negative_patterns": [ + "one-off", + "do not turn", + "without building", + "brainstorm", + "just explain", + "headline", + "summarize" + ], + "should_trigger": [ + "Convert this operations checklist into a reusable skill.", + "Add trigger evals to this skill before sharing it with the team.", + "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks." + ], + "should_not_trigger": [ + "Give me ideas for improving our process.", + "Review this note and tell me what it means in plain English.", + "Help me brainstorm several ways to improve packaging, but do not generate any skill files." + ], + "near_neighbor": [ + "Review this process note and explain it, no packaging needed.", + "Turn this into a checklist and template, but stop short of making a full skill.", + "Package this explanation as a document, not as an agent skill." + ] +} diff --git a/evals/holdout/trigger_cases.json b/evals/holdout/trigger_cases.json new file mode 100644 index 00000000..bf791bd5 --- /dev/null +++ b/evals/holdout/trigger_cases.json @@ -0,0 +1,23 @@ +{ + "recommended_threshold": 0.37, + "negative_patterns": [ + "one-off", + "do not turn", + "without building", + "brainstorm", + "just explain", + "headline", + "summarize" + ], + "should_trigger": [ + "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution." + ], + "should_not_trigger": [ + "I pasted a long process description below, but I only want a summary, not a reusable skill." + ], + "near_neighbor": [ + "Help me shape an idea before we decide whether to build a skill.", + "Create an outline for a possible future skill, but do not build the skill yet." + ] +} diff --git a/evals/train/trigger_cases.json b/evals/train/trigger_cases.json new file mode 100644 index 00000000..aa11f6cd --- /dev/null +++ b/evals/train/trigger_cases.json @@ -0,0 +1,35 @@ +{ + "recommended_threshold": 0.37, + "negative_patterns": [ + "one-off", + "do not turn", + "without building", + "brainstorm", + "just explain", + "headline", + "summarize" + ], + "should_trigger": [ + "Create a skill from this repeated workflow.", + "Improve this skill description and add evals.", + "Turn this runbook into a reusable agent skill.", + "Package this skill for team reuse.", + "Refactor this prompt into a proper skill package.", + "Create a meta-skill for packaging internal workflows." + ], + "should_not_trigger": [ + "Explain what a workflow is.", + "Write a product headline for this landing page.", + "Summarize this random note.", + "Just explain what a skill is.", + "Draft a blog title for this article.", + "Translate this README into Japanese." + ], + "near_neighbor": [ + "Create a one-off prompt for this task.", + "Help me brainstorm process ideas without building a skill.", + "Improve this README but do not turn it into a skill.", + "Make a checklist for this task, but not a reusable skill.", + "Write a custom answer for this request without creating a skill package." + ] +} diff --git a/examples/README.md b/examples/README.md index ddcb77ec..97479134 100644 --- a/examples/README.md +++ b/examples/README.md @@ -13,3 +13,7 @@ Each example contains: - `raw-workflow.md`: what the user originally gives - `design-summary.md`: how the meta-skill scopes and structures the result - `generated-skill/`: the resulting skill package shape + +Additional evolution example: + +- `evolution-frontend-review/`: shows raw workflow, v0, v1, final, and eval delta diff --git a/examples/evolution-frontend-review/README.md b/examples/evolution-frontend-review/README.md new file mode 100644 index 00000000..394d5ee6 --- /dev/null +++ b/examples/evolution-frontend-review/README.md @@ -0,0 +1,9 @@ +# Evolution Example: Frontend Review + +This example shows a full improvement chain: + +- `raw-workflow.md`: initial request +- `v0/`: initial skill draft +- `v1/`: refined skill draft with better trigger and output contract +- `final/`: final package shape +- `eval-delta.md`: what changed and why diff --git a/examples/evolution-frontend-review/eval-delta.md b/examples/evolution-frontend-review/eval-delta.md new file mode 100644 index 00000000..3b6aa466 --- /dev/null +++ b/examples/evolution-frontend-review/eval-delta.md @@ -0,0 +1,19 @@ +# Eval Delta + +## v0 + +- Trigger too vague +- No output contract +- High under-trigger risk + +## v1 + +- Trigger became more specific +- Added review categories +- Better routing, but output still underspecified + +## final + +- Stronger trigger coverage +- Output contract added +- Review standard became reusable for team workflows diff --git a/examples/evolution-frontend-review/final/SKILL.md b/examples/evolution-frontend-review/final/SKILL.md new file mode 100644 index 00000000..850d08cb --- /dev/null +++ b/examples/evolution-frontend-review/final/SKILL.md @@ -0,0 +1,22 @@ +--- +name: frontend-review +description: Review frontend code for accessibility, risky UI security issues, missing states, and common UX regressions. Use when asked to review React, UI, frontend components, forms, a11y, or pre-merge frontend quality. +--- + +# Frontend Review + +Review UI code with a pre-merge mindset. Findings come first. Focus on behavior regressions, accessibility gaps, risky form handling, unsafe rendering, and missing user states. + +## Workflow + +1. Identify the UI surface and user flows affected. +2. Check accessibility, security-sensitive inputs, loading states, and error states. +3. Report findings by severity with concrete code references. +4. Prefer actionable fixes over generic advice. + +## Output Contract + +- Start with findings, ordered by severity. +- Include file references for each finding. +- Call out open questions separately from confirmed issues. +- If there are no findings, say that explicitly and note residual risks or test gaps. diff --git a/examples/evolution-frontend-review/raw-workflow.md b/examples/evolution-frontend-review/raw-workflow.md new file mode 100644 index 00000000..ea5c2e8f --- /dev/null +++ b/examples/evolution-frontend-review/raw-workflow.md @@ -0,0 +1,10 @@ +# Raw Workflow + +Review our frontend code before merge for: + +- accessibility issues +- risky HTML rendering +- missing loading and error states +- bad form behavior + +We want a reusable review skill, not a one-off checklist. diff --git a/examples/evolution-frontend-review/v0/SKILL.md b/examples/evolution-frontend-review/v0/SKILL.md new file mode 100644 index 00000000..6e861f04 --- /dev/null +++ b/examples/evolution-frontend-review/v0/SKILL.md @@ -0,0 +1,11 @@ +--- +name: frontend-review +description: Review frontend code before merge. +--- + +# Frontend Review + +## Workflow + +1. Review the code. +2. Report issues. diff --git a/examples/evolution-frontend-review/v1/SKILL.md b/examples/evolution-frontend-review/v1/SKILL.md new file mode 100644 index 00000000..a7efff99 --- /dev/null +++ b/examples/evolution-frontend-review/v1/SKILL.md @@ -0,0 +1,14 @@ +--- +name: frontend-review +description: Review frontend code for accessibility, risky UI security issues, missing loading and error states, and pre-merge regressions. Use when asked to review React, UI, forms, frontend components, or merge-readiness for the frontend. +--- + +# Frontend Review + +Review UI code with a pre-merge mindset. + +## Workflow + +1. Identify affected flows. +2. Check accessibility, rendering safety, form behavior, loading states, and error states. +3. Report findings by severity with code references. diff --git a/scripts/diff_eval.py b/scripts/diff_eval.py new file mode 100644 index 00000000..65bffd00 --- /dev/null +++ b/scripts/diff_eval.py @@ -0,0 +1,25 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser(description="Diff two eval reports.") + parser.add_argument("before") + parser.add_argument("after") + args = parser.parse_args() + + before = json.loads(Path(args.before).read_text(encoding="utf-8")) + after = json.loads(Path(args.after).read_text(encoding="utf-8")) + diff = { + "false_positive_delta": after["summary"]["false_positives"] - before["summary"]["false_positives"], + "false_negative_delta": after["summary"]["false_negatives"] - before["summary"]["false_negatives"], + "precision_delta": round(after["summary"]["average_precision"] - before["summary"]["average_precision"], 3), + "recall_delta": round(after["summary"]["average_recall"] - before["summary"]["average_recall"], 3), + } + print(json.dumps(diff, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/init_skill.py b/scripts/init_skill.py new file mode 100644 index 00000000..32f41dd8 --- /dev/null +++ b/scripts/init_skill.py @@ -0,0 +1,56 @@ +#!/usr/bin/env python3 +import argparse +from pathlib import Path + + +SKILL_TEMPLATE = """--- +name: {name} +description: {description} +--- + +# {title} + +## Workflow + +1. Understand the request. +2. Execute the main task. +3. Validate the result. +""" + + +INTERFACE_TEMPLATE = """interface: + display_name: "{title}" + short_description: "{short_description}" + default_prompt: "Use ${name} to ..." +compatibility: + canonical_format: "agent-skills" + adapter_targets: + - "openai" + - "claude" + - "generic" +""" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Initialize a minimal skill package.") + parser.add_argument("name", help="skill folder and frontmatter name") + parser.add_argument("--description", default="Describe what the skill does and when to use it.") + parser.add_argument("--title", default=None) + parser.add_argument("--output-dir", default=".") + args = parser.parse_args() + + title = args.title or args.name.replace("-", " ").title() + root = Path(args.output_dir).resolve() / args.name + (root / "agents").mkdir(parents=True, exist_ok=True) + (root / "references").mkdir(exist_ok=True) + (root / "scripts").mkdir(exist_ok=True) + (root / "SKILL.md").write_text(SKILL_TEMPLATE.format(name=args.name, description=args.description, title=title), encoding="utf-8") + (root / "agents" / "interface.yaml").write_text( + INTERFACE_TEMPLATE.format(name=args.name, title=title, short_description=args.description[:80]), + encoding="utf-8", + ) + print(str(root)) + + +if __name__ == "__main__": + main() diff --git a/scripts/lint_skill.py b/scripts/lint_skill.py new file mode 100644 index 00000000..13fcc880 --- /dev/null +++ b/scripts/lint_skill.py @@ -0,0 +1,33 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + + +def main() -> None: + parser = argparse.ArgumentParser(description="Lint a skill package for basic structure issues.") + parser.add_argument("skill_dir") + args = parser.parse_args() + + root = Path(args.skill_dir).resolve() + failures = [] + warnings = [] + + if root.name.lower() != root.name: + warnings.append("Skill directory is not lowercase.") + if " " in root.name: + failures.append("Skill directory contains spaces.") + + skill_md = root / "SKILL.md" + if skill_md.exists(): + lines = skill_md.read_text(encoding="utf-8").splitlines() + if len(lines) > 300: + warnings.append("SKILL.md is getting long; consider moving detail into references/.") + + print(json.dumps({"ok": not failures, "failures": failures, "warnings": warnings}, ensure_ascii=False, indent=2)) + if failures: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/run_eval_suite.py b/scripts/run_eval_suite.py new file mode 100644 index 00000000..2134e6e8 --- /dev/null +++ b/scripts/run_eval_suite.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import argparse +import json +import subprocess +import sys +from pathlib import Path + + +def run_case(script: Path, description: Path, baseline: Path, cases: Path) -> dict: + proc = subprocess.run( + [ + sys.executable, + str(script), + "--description-file", + str(description), + "--baseline-description-file", + str(baseline), + "--cases", + str(cases), + ], + capture_output=True, + text=True, + ) + payload = json.loads(proc.stdout) + payload["returncode"] = proc.returncode + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Run trigger evaluation across train/dev/holdout suites.") + parser.add_argument("--eval-dir", default="evals", help="Root eval directory") + parser.add_argument("--description-file", default="evals/improved_description.txt") + parser.add_argument("--baseline-description-file", default="evals/baseline_description.txt") + args = parser.parse_args() + + root = Path(args.eval_dir).resolve() + script = Path("scripts/trigger_eval.py").resolve() + description = Path(args.description_file).resolve() + baseline = Path(args.baseline_description_file).resolve() + + suites = {} + aggregate = {"false_positives": 0, "false_negatives": 0, "precision": [], "recall": []} + for name in ("train", "dev", "holdout"): + report = run_case(script, description, baseline, root / name / "trigger_cases.json") + suites[name] = report + aggregate["false_positives"] += report["false_positives"] + aggregate["false_negatives"] += report["false_negatives"] + if report["precision"] is not None: + aggregate["precision"].append(report["precision"]) + if report["recall"] is not None: + aggregate["recall"].append(report["recall"]) + + summary = { + "suite_count": len(suites), + "false_positives": aggregate["false_positives"], + "false_negatives": aggregate["false_negatives"], + "average_precision": round(sum(aggregate["precision"]) / len(aggregate["precision"]), 3) if aggregate["precision"] else None, + "average_recall": round(sum(aggregate["recall"]) / len(aggregate["recall"]), 3) if aggregate["recall"] else None, + } + output = {"summary": summary, "suites": suites} + print(json.dumps(output, ensure_ascii=False, indent=2)) + if summary["false_positives"] > 0 or summary["false_negatives"] > 0: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py new file mode 100644 index 00000000..6ef9e1c7 --- /dev/null +++ b/scripts/validate_skill.py @@ -0,0 +1,47 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path +import yaml + + +def main() -> None: + parser = argparse.ArgumentParser(description="Validate a skill package.") + parser.add_argument("skill_dir") + args = parser.parse_args() + + root = Path(args.skill_dir).resolve() + failures = [] + skill_md = root / "SKILL.md" + interface = root / "agents" / "interface.yaml" + + if not skill_md.exists(): + failures.append("Missing SKILL.md") + if not interface.exists(): + failures.append("Missing agents/interface.yaml") + + if skill_md.exists(): + text = skill_md.read_text(encoding="utf-8") + if not text.startswith("---"): + failures.append("SKILL.md missing frontmatter") + else: + parts = text.split("---", 2) + data = yaml.safe_load(parts[1]) or {} + for field in ("name", "description"): + if not data.get(field): + failures.append(f"Missing frontmatter field: {field}") + + if interface.exists(): + data = yaml.safe_load(interface.read_text(encoding="utf-8")) or {} + meta = data.get("interface", {}) + for field in ("display_name", "short_description", "default_prompt"): + if not meta.get(field): + failures.append(f"Missing interface field: {field}") + + print(json.dumps({"ok": not failures, "failures": failures}, ensure_ascii=False, indent=2)) + if failures: + raise SystemExit(2) + + +if __name__ == "__main__": + main()