diff --git a/.gitignore b/.gitignore index fe954a4..cc3317b 100644 --- a/.gitignore +++ b/.gitignore @@ -3,3 +3,4 @@ dist/ *.zip __pycache__/ tests/tmp/ +tests/tmp_snapshot/ diff --git a/Makefile b/Makefile index 3fc9e88..33bbe36 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite package-check package-failure-check validate lint test clean +.PHONY: eval eval-suite package-check package-failure-check snapshot-check validate lint test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -14,13 +14,16 @@ package-check: package-failure-check: $(PYTHON) tests/verify_packager_failures.py +snapshot-check: + $(PYTHON) tests/verify_adapter_snapshots.py + validate: $(PYTHON) scripts/validate_skill.py . lint: $(PYTHON) scripts/lint_skill.py . -test: eval eval-suite package-check package-failure-check validate lint +test: eval eval-suite package-check package-failure-check snapshot-check validate lint clean: - rm -rf dist tests/tmp + rm -rf dist tests/tmp tests/tmp_snapshot diff --git a/README.md b/README.md index edc8980..fdd57cd 100644 --- a/README.md +++ b/README.md @@ -44,6 +44,14 @@ Or run everything together: make test ``` +## Results + +- `make test` passes locally +- trigger eval: `0` false positives, `0` false negatives on the current regression set +- eval suite: train / dev / holdout all pass +- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks +- packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected + ## What It Does This project helps you create, refactor, evaluate, and package skills as durable capability bundles rather than one-off prompts. @@ -198,7 +206,9 @@ This project is best for: - Examples: [examples/README.md](examples/README.md) - Evals: [evals/README.md](evals/README.md) - Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md) +- Platform capability matrix: [references/platform-capability-matrix.md](references/platform-capability-matrix.md) - Failure fixtures: [tests/fixtures](tests/fixtures) +- Adapter snapshots: [tests/snapshots](tests/snapshots) - Evolution example: [examples/evolution-frontend-review/README.md](examples/evolution-frontend-review/README.md) ## License diff --git a/docs/README.fr-FR.md b/docs/README.fr-FR.md index 651467a..687ec41 100644 --- a/docs/README.fr-FR.md +++ b/docs/README.fr-FR.md @@ -15,6 +15,13 @@ Il transforme des workflows bruts, des transcripts, des prompts, des notes et de 2. Utilisez `yao-meta-skill` pour générer ou améliorer le paquet en mode scaffold, production ou library. 3. Exécutez `context_sizer.py`, `trigger_eval.py` et `cross_packager.py` selon le besoin pour valider et exporter le résultat. +## Results + +- `make test` passe actuellement +- sur le jeu de régression courant, trigger eval a `0` faux positifs et `0` faux négatifs +- les suites train / dev / holdout passent toutes +- les contrats de packaging `openai`, `claude` et `generic` sont validés + ## Ce que fait le projet Ce projet permet de créer, refactoriser, évaluer et empaqueter des skills comme des briques de capacité durables plutôt que comme des prompts ponctuels. diff --git a/docs/README.ja-JP.md b/docs/README.ja-JP.md index 0f1c12c..7b3fdf3 100644 --- a/docs/README.ja-JP.md +++ b/docs/README.ja-JP.md @@ -15,6 +15,13 @@ 2. `yao-meta-skill` を使って scaffold、production、library のいずれかのモードでパッケージを生成または改善します。 3. 必要に応じて `context_sizer.py`、`trigger_eval.py`、`cross_packager.py` を実行し、検証と出力を行います。 +## Results + +- 現在 `make test` は通過 +- 現在の回帰セットでは trigger eval が `0` false positives / `0` false negatives +- train / dev / holdout の 3 層評価が通過 +- `openai`、`claude`、`generic` の packaging contract が通過 + ## 何をするものか このプロジェクトは、skill を単発の prompt ではなく、作成・改善・評価・配布できる持続的な能力パッケージとして扱えるようにします。 diff --git a/docs/README.ru-RU.md b/docs/README.ru-RU.md index b46a145..4325f00 100644 --- a/docs/README.ru-RU.md +++ b/docs/README.ru-RU.md @@ -15,6 +15,13 @@ 2. Используйте `yao-meta-skill`, чтобы сгенерировать или улучшить пакет в режиме scaffold, production или library. 3. При необходимости запустите `context_sizer.py`, `trigger_eval.py` и `cross_packager.py`, чтобы проверить и экспортировать результат. +## Results + +- `make test` сейчас проходит +- на текущем regression-наборе trigger eval дает `0` false positives и `0` false negatives +- все три набора train / dev / holdout проходят +- packaging contracts для `openai`, `claude` и `generic` проходят проверку + ## Что делает проект Этот проект помогает создавать, перерабатывать, оценивать и упаковывать skills как долговечные capability-пакеты, а не как одноразовые prompts. diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index f40d6af..b219a6a 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -15,6 +15,13 @@ 2. 使用 `yao-meta-skill` 以 scaffold、production 或 library 模式生成或改进 skill 包。 3. 按需要运行 `context_sizer.py`、`trigger_eval.py` 和 `cross_packager.py` 来检查并导出结果。 +## Results + +- 当前 `make test` 可通过 +- 当前回归集下 trigger eval 为 `0` 误触发、`0` 漏触发 +- train / dev / holdout 三层评测均通过 +- `openai`、`claude`、`generic` 三个目标的 packaging contract 校验通过 + ## 它能做什么 这个项目帮助你把 skill 从一次性 prompt,升级成可创建、可重构、可评估、可打包的长期能力包。 diff --git a/references/platform-capability-matrix.md b/references/platform-capability-matrix.md new file mode 100644 index 0000000..e95d1e6 --- /dev/null +++ b/references/platform-capability-matrix.md @@ -0,0 +1,31 @@ +# Platform Capability Matrix + +This matrix describes the current packaging targets and their support level. + +| Target | Metadata Adapter | Output Contract | Snapshot Test | Notes | +| --- | --- | --- | --- | --- | +| `openai` | Yes | Yes | Yes | Generates `targets/openai/agents/openai.yaml` | +| `claude` | Yes | Yes | Yes | Generates `targets/claude/README.md` plus adapter metadata | +| `generic` | Yes | Yes | Yes | Uses neutral adapter metadata only | + +## Current Support Model + +- `openai`: strongest metadata adapter support +- `claude`: lightweight compatibility adapter, behavior still relies mainly on neutral source files +- `generic`: lowest-friction export for neutral Agent Skills consumers + +## Explicit Non-Goals + +This project does not yet implement: + +- runtime-specific behavior transforms +- client SDK integration +- provider-specific execution logic + +## Degradation Rule + +If a target cannot support a source feature directly: + +1. preserve the neutral source package +2. emit a minimal adapter +3. document the fallback in the target output diff --git a/tests/snapshots/claude_adapter.json b/tests/snapshots/claude_adapter.json new file mode 100644 index 0000000..4ba9f56 --- /dev/null +++ b/tests/snapshots/claude_adapter.json @@ -0,0 +1,5 @@ +{ + "platform": "claude", + "canonical_metadata": "agents/interface.yaml", + "required_generated_file": "targets/claude/README.md" +} diff --git a/tests/snapshots/generic_adapter.json b/tests/snapshots/generic_adapter.json new file mode 100644 index 0000000..7727b9b --- /dev/null +++ b/tests/snapshots/generic_adapter.json @@ -0,0 +1,5 @@ +{ + "platform": "generic", + "canonical_metadata": "agents/interface.yaml", + "required_generated_file": "targets/generic/adapter.json" +} diff --git a/tests/snapshots/openai_adapter.json b/tests/snapshots/openai_adapter.json new file mode 100644 index 0000000..0a0691e --- /dev/null +++ b/tests/snapshots/openai_adapter.json @@ -0,0 +1,5 @@ +{ + "platform": "openai", + "canonical_metadata": "agents/interface.yaml", + "required_generated_file": "targets/openai/agents/openai.yaml" +} diff --git a/tests/verify_adapter_snapshots.py b/tests/verify_adapter_snapshots.py new file mode 100644 index 0000000..c294438 --- /dev/null +++ b/tests/verify_adapter_snapshots.py @@ -0,0 +1,62 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "cross_packager.py" +EXPECTATIONS = ROOT / "evals" / "packaging_expectations.json" +SNAPSHOTS = ROOT / "tests" / "snapshots" +TMP = ROOT / "tests" / "tmp_snapshot" + + +def main() -> None: + if TMP.exists(): + shutil.rmtree(TMP) + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--expectations", + str(EXPECTATIONS), + "--output-dir", + str(TMP), + ], + cwd=ROOT, + capture_output=True, + text=True, + ) + if proc.returncode != 0: + print(proc.stdout) + print(proc.stderr) + raise SystemExit(proc.returncode) + + failures = [] + for name in ("openai", "claude", "generic"): + snapshot = json.loads((SNAPSHOTS / f"{name}_adapter.json").read_text(encoding="utf-8")) + adapter = json.loads((TMP / "targets" / name / "adapter.json").read_text(encoding="utf-8")) + if adapter.get("platform") != snapshot["platform"]: + failures.append(f"{name}: platform mismatch") + if adapter.get("canonical_metadata") != snapshot["canonical_metadata"]: + failures.append(f"{name}: canonical metadata mismatch") + if not (TMP / snapshot["required_generated_file"]).exists(): + failures.append(f"{name}: missing generated file {snapshot['required_generated_file']}") + + report = {"ok": not failures, "failures": failures} + print(json.dumps(report, ensure_ascii=False, indent=2)) + if failures: + raise SystemExit(2) + + +if __name__ == "__main__": + main()