diff --git a/Makefile b/Makefile new file mode 100644 index 00000000..1c342811 --- /dev/null +++ b/Makefile @@ -0,0 +1,17 @@ +PYTHON ?= python3 + +.PHONY: eval package-check package-failure-check test clean + +eval: + $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt + +package-check: + $(PYTHON) scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --output-dir dist --zip + +package-failure-check: + $(PYTHON) tests/verify_packager_failures.py + +test: eval package-check package-failure-check + +clean: + rm -rf dist tests/tmp diff --git a/README.md b/README.md index 741a3b2e..e0991217 100644 --- a/README.md +++ b/README.md @@ -35,6 +35,13 @@ Minimum commands: python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json python3 scripts/context_sizer.py . python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip +python3 tests/verify_packager_failures.py +``` + +Or run everything together: + +```bash +make test ``` ## What It Does @@ -113,6 +120,7 @@ Three end-to-end examples showing raw workflow input, design summary, and final - The sample trigger report now covers a larger positive, negative, and near-neighbor set rather than a tiny demo set. - Packaging validation now uses explicit contracts and YAML parsing, but it is still a lightweight local validation layer rather than a full platform integration suite. - `evals/failure-cases.md` captures known weak spots that should remain part of regression checks. +- `tests/verify_packager_failures.py` checks that invalid metadata, invalid YAML, and unsupported targets fail clearly. ### `templates/` @@ -183,6 +191,7 @@ This project is best for: - Examples: [examples/README.md](examples/README.md) - Evals: [evals/README.md](evals/README.md) - Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md) +- Failure fixtures: [tests/fixtures](tests/fixtures) ## License diff --git a/evals/README.md b/evals/README.md index 29402826..553f6f7b 100644 --- a/evals/README.md +++ b/evals/README.md @@ -17,4 +17,14 @@ Use: python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --threshold 0.35 python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt --threshold 0.35 python3 scripts/cross_packager.py . --platform openai --platform claude --expectations evals/packaging_expectations.json --zip +python3 tests/verify_packager_failures.py ``` + +Regression scope now includes: + +- direct positives +- direct negatives +- near neighbors +- long-context positives +- mixed-intent negatives +- explicit "do not build a skill" negatives diff --git a/evals/trigger_cases.json b/evals/trigger_cases.json index b4d06cf2..6c992637 100644 --- a/evals/trigger_cases.json +++ b/evals/trigger_cases.json @@ -1,5 +1,5 @@ { - "recommended_threshold": 0.42, + "recommended_threshold": 0.37, "negative_patterns": [ "one-off", "do not turn", @@ -17,7 +17,10 @@ "Convert this operations checklist into a reusable skill.", "Refactor this prompt into a proper skill package.", "Add trigger evals to this skill before sharing it with the team.", - "Create a meta-skill for packaging internal workflows." + "Create a meta-skill for packaging internal workflows.", + "We have a messy release runbook, export process, and a prompt history; turn all of that into one reusable skill package with evals and packaging checks.", + "Build a reusable skill from this long internal process note, and make sure it has references, scripts, and a trigger description.", + "Take this existing skill draft, tighten the trigger boundary, add near-neighbor evals, and package it for distribution." ], "should_not_trigger": [ "Explain what a workflow is.", @@ -26,7 +29,10 @@ "Just explain what a skill is.", "Draft a blog title for this article.", "Translate this README into Japanese.", - "Give me ideas for improving our process." + "Give me ideas for improving our process.", + "I pasted a long process description below, but I only want a summary, not a reusable skill.", + "Review this note and tell me what it means in plain English.", + "Help me brainstorm several ways to improve packaging, but do not generate any skill files." ], "near_neighbor": [ "Create a one-off prompt for this task.", @@ -35,6 +41,9 @@ "Make a checklist for this task, but not a reusable skill.", "Review this process note and explain it, no packaging needed.", "Help me shape an idea before we decide whether to build a skill.", - "Write a custom answer for this request without creating a skill package." + "Write a custom answer for this request without creating a skill package.", + "Turn this into a checklist and template, but stop short of making a full skill.", + "Package this explanation as a document, not as an agent skill.", + "Create an outline for a possible future skill, but do not build the skill yet." ] } diff --git a/scripts/cross_packager.py b/scripts/cross_packager.py index aa19774a..363a6bf3 100644 --- a/scripts/cross_packager.py +++ b/scripts/cross_packager.py @@ -197,10 +197,11 @@ def main() -> None: shutil.rmtree(out_dir) out_dir.mkdir(parents=True) - manifest = copy_manifest(skill_dir, out_dir) - generated = [str(manifest)] + generated = [] failures = [] try: + manifest = copy_manifest(skill_dir, out_dir) + generated.append(str(manifest)) for platform in (args.platform or ["generic"]): generated.append(str(write_adapter(skill_dir, out_dir, platform))) if args.zip: diff --git a/tests/fixtures/package_invalid_yaml/SKILL.md b/tests/fixtures/package_invalid_yaml/SKILL.md new file mode 100644 index 00000000..1c42887b --- /dev/null +++ b/tests/fixtures/package_invalid_yaml/SKILL.md @@ -0,0 +1,6 @@ +--- +name: broken-yaml-skill +description: Broken YAML fixture for packager failure testing. +--- + +# Broken YAML Skill diff --git a/tests/fixtures/package_invalid_yaml/agents/interface.yaml b/tests/fixtures/package_invalid_yaml/agents/interface.yaml new file mode 100644 index 00000000..c1f52b6a --- /dev/null +++ b/tests/fixtures/package_invalid_yaml/agents/interface.yaml @@ -0,0 +1,8 @@ +interface: + display_name: "Broken YAML" + short_description: "This fixture contains invalid yaml" + default_prompt: "Use this fixture" +compatibility: + canonical_format: "agent-skills" + adapter_targets: + - "openai diff --git a/tests/fixtures/package_missing_interface_field/SKILL.md b/tests/fixtures/package_missing_interface_field/SKILL.md new file mode 100644 index 00000000..282c71e2 --- /dev/null +++ b/tests/fixtures/package_missing_interface_field/SKILL.md @@ -0,0 +1,6 @@ +--- +name: broken-skill +description: Broken fixture for packager failure testing. +--- + +# Broken Skill diff --git a/tests/fixtures/package_missing_interface_field/agents/interface.yaml b/tests/fixtures/package_missing_interface_field/agents/interface.yaml new file mode 100644 index 00000000..f0f06b22 --- /dev/null +++ b/tests/fixtures/package_missing_interface_field/agents/interface.yaml @@ -0,0 +1,7 @@ +interface: + display_name: "Broken Skill" + short_description: "Missing default prompt on purpose" +compatibility: + canonical_format: "agent-skills" + adapter_targets: + - "openai" diff --git a/tests/tmp/unsupported_platform/manifest.json b/tests/tmp/unsupported_platform/manifest.json new file mode 100644 index 00000000..c4648ab4 --- /dev/null +++ b/tests/tmp/unsupported_platform/manifest.json @@ -0,0 +1,16 @@ +{ + "name": "yao-meta-skill", + "description": "Create, refactor, evaluate, and package agent skills from workflows, transcripts, prompts, docs, or rough notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, optimize skill triggering, add evals, or prepare a skill for team reuse. Relevant for meta-skills, skill factories, skill templates, skill QA, and cross-platform skill packaging.", + "version": "1.0.0", + "platform": "generic", + "skill_root": "yao-meta-skill", + "display_name": "Yao Meta Skill", + "short_description": "Create robust, trigger-aware agent skills", + "default_prompt": "Use $yao-meta-skill to turn my workflow, prompt set, or rough notes into a production-ready skill with strong triggering, lean structure, and an eval plan.", + "canonical_metadata": "agents/interface.yaml", + "adapter_targets": [ + "openai", + "claude", + "generic" + ] +} \ No newline at end of file diff --git a/tests/verify_packager_failures.py b/tests/verify_packager_failures.py new file mode 100644 index 00000000..be8f0142 --- /dev/null +++ b/tests/verify_packager_failures.py @@ -0,0 +1,91 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "cross_packager.py" +EXPECTATIONS = ROOT / "evals" / "packaging_expectations.json" +TMP = ROOT / "tests" / "tmp" + + +def run_case(name: str, cmd: list[str], expected_substring: str) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = {} + if proc.stdout.strip(): + try: + payload = json.loads(proc.stdout) + except json.JSONDecodeError: + payload = {"raw_stdout": proc.stdout} + joined = proc.stdout + "\n" + proc.stderr + passed = proc.returncode == 2 and expected_substring in joined + return { + "name": name, + "passed": passed, + "returncode": proc.returncode, + "expected_substring": expected_substring, + "stdout": proc.stdout, + "stderr": proc.stderr, + "payload": payload, + } + + +def main() -> None: + TMP.mkdir(parents=True, exist_ok=True) + cases = [ + run_case( + "missing_interface_field", + [ + sys.executable, + str(SCRIPT), + str(ROOT / "tests" / "fixtures" / "package_missing_interface_field"), + "--platform", + "openai", + "--expectations", + str(EXPECTATIONS), + "--output-dir", + str(TMP / "missing_interface_field"), + ], + "Missing required interface fields", + ), + run_case( + "invalid_yaml", + [ + sys.executable, + str(SCRIPT), + str(ROOT / "tests" / "fixtures" / "package_invalid_yaml"), + "--platform", + "openai", + "--expectations", + str(EXPECTATIONS), + "--output-dir", + str(TMP / "invalid_yaml"), + ], + "while scanning a quoted scalar", + ), + run_case( + "unsupported_platform", + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--platform", + "bad_target", + "--expectations", + str(EXPECTATIONS), + "--output-dir", + str(TMP / "unsupported_platform"), + ], + "Unsupported platform", + ), + ] + report = {"ok": all(case["passed"] for case in cases), "cases": cases} + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main()