diff --git a/.gitignore b/.gitignore index 0f85381..8c4e8bd 100644 --- a/.gitignore +++ b/.gitignore @@ -7,4 +7,5 @@ tests/tmp_snapshot/ tests/tmp_cli/ tests/tmp_skill_overview/ tests/tmp_skill_preview/ +tests/tmp_reference_scan/ reports/release_snapshots/ diff --git a/Makefile b/Makefile index a605e59..92cf944 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check reference-scan-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -32,6 +32,9 @@ yao-cli-check: skill-overview-check: $(PYTHON) tests/verify_skill_overview.py +reference-scan-check: + $(PYTHON) tests/verify_reference_scan.py + description-drift-history: $(PYTHON) scripts/render_description_drift_history.py @@ -80,7 +83,7 @@ resource-boundary-check: quality-check: $(PYTHON) tests/verify_quality_checks.py -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check reference-scan-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check clean: rm -rf dist tests/tmp tests/tmp_snapshot diff --git a/README.md b/README.md index 837e326..fc7f55f 100644 --- a/README.md +++ b/README.md @@ -19,6 +19,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a clear trigger surface - a lean `SKILL.md` - optional references, scripts, and evals +- a controlled benchmark scan before deep authoring - a generated visual HTML overview for each newly initialized skill - neutral source metadata plus client-specific adapters - governance, promotion, and portability checks built into the default flow @@ -46,13 +47,15 @@ Read it in 10 seconds: ## Quick Start 1. Describe the workflow, prompt set, or repeated task you want to turn into a skill. -2. Use `yao-meta-skill` to generate or improve the package in scaffold, production, or library mode. -3. Run `context_sizer.py`, `resource_boundary_check.py`, `governance_check.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result. +2. Run a short reference scan so the package can borrow strong patterns without copying extra weight. +3. Use `yao-meta-skill` to generate or improve the package in scaffold, production, or library mode. +4. Run `context_sizer.py`, `resource_boundary_check.py`, `governance_check.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result. Or use the unified authoring CLI: ```bash python3 scripts/yao.py init my-skill --description "Describe what the skill does." +python3 scripts/yao.py reference-scan my-skill --reference "World Class Method::method::Borrow a tight evaluation loop.::Do not copy heavy process." python3 scripts/yao.py skill-report my-skill python3 scripts/yao.py package . --platform generic --output-dir dist ``` @@ -196,6 +199,7 @@ The design logic is simple: The repository now treats method as a first-class asset instead of scattered guidance. - [Skill Engineering Method](references/skill-engineering-method.md) +- [Reference Scan Strategy](references/reference-scan.md) - [Skill Archetypes](references/skill-archetypes.md) - [Gate Selection](references/gate-selection.md) - [Non-Skill Decision Tree](references/non-skill-decision-tree.md) diff --git a/SKILL.md b/SKILL.md index cb32abe..723782a 100644 --- a/SKILL.md +++ b/SKILL.md @@ -28,11 +28,12 @@ Mode rules: [Operating Modes](references/operating-modes.md), [QA Ladder](refere ## Compact Workflow 1. Decide whether the request should become a skill, then choose the lightest archetype. -2. Capture the recurring job, outputs, trigger phrases, and exclusions. -3. Write the `description` early, then test route quality before expanding the package. -4. Add only the folders and gates that earn their keep: `trigger_eval.py`, `optimize_description.py`, `judge_blind_eval.py`, `resource_boundary_check.py`, `governance_check.py`, `cross_packager.py`. +2. Run a short reference scan to borrow patterns from strong benchmark objects without copying their prose or weight. +3. Capture the recurring job, outputs, trigger phrases, and exclusions. +4. Write the `description` early, then test route quality before expanding the package. +5. Add only the folders and gates that earn their keep: `trigger_eval.py`, `optimize_description.py`, `judge_blind_eval.py`, `resource_boundary_check.py`, `governance_check.py`, `cross_packager.py`. -Playbooks: [Skill Engineering Method](references/skill-engineering-method.md), [Skill Archetypes](references/skill-archetypes.md), [Gate Selection](references/gate-selection.md), [Non-Skill Decision Tree](references/non-skill-decision-tree.md), [Operating Modes](references/operating-modes.md), [Trigger And Eval Playbook](references/eval-playbook.md). +Playbooks: [Skill Engineering Method](references/skill-engineering-method.md), [Reference Scan Strategy](references/reference-scan.md), [Skill Archetypes](references/skill-archetypes.md), [Gate Selection](references/gate-selection.md), [Non-Skill Decision Tree](references/non-skill-decision-tree.md), [Operating Modes](references/operating-modes.md), [Trigger And Eval Playbook](references/eval-playbook.md). ## Output Contract @@ -42,11 +43,12 @@ Unless the user asks otherwise, produce: 2. a trigger-aware `SKILL.md` 3. aligned `agents/interface.yaml` 4. optional `references/`, `scripts/`, `evals/`, `reports/`, and `manifest.json` only when justified -5. a short summary of boundary, exclusions, gates, and next steps +5. a short summary of boundary, exclusions, benchmark objects, gates, and next steps ## Reference Map - [Skill Engineering Method](references/skill-engineering-method.md) +- [Reference Scan Strategy](references/reference-scan.md) - [Skill Archetypes](references/skill-archetypes.md) - [Gate Selection](references/gate-selection.md) - [Non-Skill Decision Tree](references/non-skill-decision-tree.md) diff --git a/docs/README.fr-FR.md b/docs/README.fr-FR.md index 654d2ab..232fe1b 100644 --- a/docs/README.fr-FR.md +++ b/docs/README.fr-FR.md @@ -9,6 +9,7 @@ Il transforme des workflows bruts, des transcripts, des prompts, des notes et de - une surface de déclenchement claire - un `SKILL.md` léger - des references, scripts et evals optionnels +- un benchmark/reference scan contrôlé avant l'authoring profond - un rapport HTML minimaliste en fond blanc généré automatiquement pour chaque nouveau skill - des métadonnées sources neutres et des adaptateurs spécifiques au client - des contrôles de gouvernance, de promotion et de portabilité intégrés au flux standard @@ -36,8 +37,9 @@ Lecture en 10 secondes : ## Quick Start 1. Décrivez le workflow, l'ensemble de prompts ou la tâche répétée que vous voulez transformer en skill. -2. Utilisez `yao-meta-skill` pour générer ou améliorer le paquet en mode scaffold, production ou library. -3. Chaque nouveau skill reçoit aussi `reports/skill-overview.html`, pour comprendre rapidement son architecture, sa logique, son usage et ses points forts. +2. Lancez un court reference scan pour extraire les bons patterns sans copier la lourdeur des sources. +3. Utilisez `yao-meta-skill` pour générer ou améliorer le paquet en mode scaffold, production ou library. +4. Chaque nouveau skill reçoit aussi `reports/skill-overview.html` et `reports/reference-scan.md`, pour comprendre rapidement son architecture, sa logique, son usage et ses références utiles. ## Results diff --git a/docs/README.ja-JP.md b/docs/README.ja-JP.md index b7e6faf..6d04119 100644 --- a/docs/README.ja-JP.md +++ b/docs/README.ja-JP.md @@ -9,6 +9,7 @@ - 明確なトリガー面 - 軽量な `SKILL.md` - 必要に応じた references、scripts、evals +- 深い authoring の前に、制御された benchmark/reference scan を 1 回行う - 新しい skill ごとに、白背景の簡潔な HTML overview を自動生成 - 中立的なソースメタデータとクライアント別アダプタ - ガバナンス、昇格判定、portability チェックを標準フローに内蔵 @@ -36,8 +37,9 @@ flowchart LR ## Quick Start 1. skill 化したい workflow、prompt 集合、または反復タスクを説明します。 -2. `yao-meta-skill` を使って scaffold、production、library のいずれかのモードでパッケージを生成または改善します。 -3. 新しく作成した skill には `reports/skill-overview.html` が付き、構造、ロジック、使い方、強みをすぐ確認できます。 +2. まず短い reference scan を行い、借りるべき pattern だけを抽出します。 +3. `yao-meta-skill` を使って scaffold、production、library のいずれかのモードでパッケージを生成または改善します。 +4. 新しく作成した skill には `reports/skill-overview.html` と `reports/reference-scan.md` が付き、構造、ロジック、使い方、参考対象をすぐ確認できます。 ## Results diff --git a/docs/README.ru-RU.md b/docs/README.ru-RU.md index 36e67ab..792c89c 100644 --- a/docs/README.ru-RU.md +++ b/docs/README.ru-RU.md @@ -9,6 +9,7 @@ - понятной поверхностью срабатывания - компактным `SKILL.md` - необязательными references, scripts и evals +- контролируемым benchmark/reference scan до глубокого authoring - автоматически создаваемым минималистичным HTML-обзором на белом фоне для каждого нового skill - нейтральными исходными метаданными и клиентскими адаптерами - встроенными проверками governance, promotion и portability в стандартном потоке @@ -36,8 +37,9 @@ flowchart LR ## Quick Start 1. Опишите workflow, набор prompts или повторяющуюся задачу, которую хотите превратить в skill. -2. Используйте `yao-meta-skill`, чтобы сгенерировать или улучшить пакет в режиме scaffold, production или library. -3. Каждый новый skill также получает `reports/skill-overview.html`, чтобы быстро понять архитектуру, логику, способ использования и сильные стороны. +2. Сначала выполните короткий reference scan, чтобы взять сильные паттерны без копирования лишней тяжести. +3. Используйте `yao-meta-skill`, чтобы сгенерировать или улучшить пакет в режиме scaffold, production или library. +4. Каждый новый skill также получает `reports/skill-overview.html` и `reports/reference-scan.md`, чтобы быстро понять архитектуру, логику, способ использования и полезные ориентиры. ## Results diff --git a/docs/README.zh-CN.md b/docs/README.zh-CN.md index 33f2ee6..aa9203f 100644 --- a/docs/README.zh-CN.md +++ b/docs/README.zh-CN.md @@ -9,6 +9,7 @@ - 清晰的触发面 - 精简的 `SKILL.md` - 可选的 references、scripts 和 evals +- 深度起草前先做一轮受控的 benchmark/reference scan - 新建 skill 时自动生成一份极简白底 HTML 可视化说明 - 中性的源元数据以及面向不同客户端的适配层 - 内建的治理、晋升和 portability 检查 @@ -36,8 +37,9 @@ flowchart LR ## Quick Start 1. 先描述你想沉淀成 skill 的 workflow、prompt 集合或重复任务。 -2. 使用 `yao-meta-skill` 以 scaffold、production 或 library 模式生成或改进 skill 包。 -3. 新建 skill 后,会默认附带 `reports/skill-overview.html`,方便快速理解架构、逻辑、使用方法和优势。 +2. 先做一轮简短的参考扫描,提炼可借鉴模式,而不是直接复制现成做法。 +3. 使用 `yao-meta-skill` 以 scaffold、production 或 library 模式生成或改进 skill 包。 +4. 新建 skill 后,会默认附带 `reports/skill-overview.html` 和 `reports/reference-scan.md`,方便理解架构、逻辑、使用方法和可借鉴对象。 ## Results diff --git a/references/reference-scan.md b/references/reference-scan.md new file mode 100644 index 0000000..79f81e9 --- /dev/null +++ b/references/reference-scan.md @@ -0,0 +1,78 @@ +# Reference Scan Strategy + +Use a short benchmark scan before deep authoring. The goal is to borrow durable patterns from strong reference objects without importing their prose, weight, or brand language into the new skill. + +## Why This Step Exists + +A new skill often fails because it starts from an isolated idea instead of a proven pattern. A controlled reference scan improves the package before it grows: + +- better boundary design +- cleaner folder and metadata choices +- more realistic quality gates +- stronger portability decisions + +## The Rule + +Reference scanning is mandatory for: + +- `Production` skills +- `Library` skills +- `Governed` skills +- meta-skills or packaging-heavy skills + +Reference scanning is optional for: + +- `Scaffold` skills +- one-person exploratory skills + +## Scope Limit + +Do not turn this into open-ended research. + +- scan at most `3-5` reference objects +- pick from no more than `3` categories +- extract patterns, not long copied content +- stop as soon as the borrow plan is clear + +## Reference Categories + +Choose the smallest relevant set: + +- `method`: loops, evaluation discipline, iteration structure +- `structure`: package anatomy, resource boundaries, metadata patterns +- `execution`: operator flow, scripts, initialization and validation experience +- `portability`: neutral metadata, adapters, degradation strategy +- `domain`: workflow-specific patterns from a top example in the same problem space + +## Output Format + +A good scan produces a short report with: + +1. current skill anchor +2. scan focus +3. `3-5` reference objects at most +4. what to borrow +5. what not to borrow +6. a compact borrow plan + +## What To Borrow + +Borrow: + +- repeatable loops +- clear boundary patterns +- proven gate choices +- portable metadata ideas +- clear operator-facing flows + +Do not borrow: + +- source-specific branding +- long copied prose +- unnecessary directories +- quality gates that exceed the skill's risk tier +- platform lock-in disguised as best practice + +## Design Principle + +The scan is successful only if it raises skill quality faster than it raises context cost. If benchmark material makes the new skill heavier without making it more reliable, discard it. diff --git a/references/skill-engineering-method.md b/references/skill-engineering-method.md index d326dfa..fab4326 100644 --- a/references/skill-engineering-method.md +++ b/references/skill-engineering-method.md @@ -53,7 +53,18 @@ Every skill should answer four questions clearly: Boundary work comes before polishing prose. -## Phase 4: Trigger-First Authoring +## Phase 4: Reference Scan + +Run a short benchmark pass before deep authoring. + +- scan `3-5` reference objects at most +- choose from method, structure, execution, portability, and domain patterns +- extract only what improves reliability or clarity +- record what not to borrow so the new skill stays light + +See [Reference Scan Strategy](reference-scan.md). + +## Phase 5: Trigger-First Authoring Author the frontmatter `description` before expanding the body. @@ -71,7 +82,7 @@ Trigger quality is improved through: - adversarial holdout - route confusion -## Phase 5: Gate Selection +## Phase 6: Gate Selection Add gates by risk, not by habit. @@ -82,7 +93,7 @@ Add gates by risk, not by habit. See [Gate Selection](gate-selection.md). -## Phase 6: Promotion +## Phase 7: Promotion A candidate route or package is promotable only when: diff --git a/scripts/init_skill.py b/scripts/init_skill.py index b4df981..15b0074 100644 --- a/scripts/init_skill.py +++ b/scripts/init_skill.py @@ -4,6 +4,7 @@ import json from datetime import date from pathlib import Path +from render_reference_scan import render_reference_scan from render_skill_overview import render_skill_overview @@ -82,7 +83,7 @@ def build_manifest(name: str) -> dict: "maturity_tier": "scaffold", "lifecycle_stage": "scaffold", "context_budget_tier": "scaffold", - "review_cadence": "ad-hoc", + "review_cadence": "per-release", "target_platforms": [ "openai", "claude", @@ -130,6 +131,7 @@ def main() -> None: encoding="utf-8", ) overview = render_skill_overview(root) + reference_scan = render_reference_scan(root, []) print( json.dumps( { @@ -139,6 +141,7 @@ def main() -> None: "readme": str(root / "README.md"), "manifest": str(root / "manifest.json"), **overview["artifacts"], + **reference_scan["artifacts"], }, }, ensure_ascii=False, diff --git a/scripts/render_reference_scan.py b/scripts/render_reference_scan.py new file mode 100644 index 0000000..d9f9ea0 --- /dev/null +++ b/scripts/render_reference_scan.py @@ -0,0 +1,214 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + +try: + import yaml +except ImportError: # pragma: no cover + yaml = None + + +def parse_frontmatter(text: str) -> tuple[dict, str]: + lines = text.splitlines() + if not lines or lines[0].strip() != "---": + return {}, text + try: + end_index = lines[1:].index("---") + 1 + except ValueError: + return {}, text + frontmatter = "\n".join(lines[1:end_index]) + body = "\n".join(lines[end_index + 1 :]).lstrip() + if yaml is not None: + payload = yaml.safe_load(frontmatter) or {} + return payload if isinstance(payload, dict) else {}, body + data = {} + for line in frontmatter.splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + data[key.strip()] = value.strip().strip('"') + return data, body + + +def extract_title(body: str, fallback: str) -> str: + for line in body.splitlines(): + if line.startswith("# "): + return line[2:].strip() + return fallback + + +def parse_reference(value: str) -> dict: + parts = [part.strip() for part in value.split("::")] + while len(parts) < 4: + parts.append("") + name, category, borrow, avoid = parts[:4] + return { + "name": name or "Unnamed reference", + "category": category or "general", + "borrow": borrow or "Capture the reusable pattern, not the prose.", + "avoid": avoid or "Do not copy source-specific language or unnecessary weight.", + } + + +def infer_scan_focus(skill_dir: Path, description: str) -> list[dict]: + checks = [] + if (skill_dir / "evals").exists(): + checks.append( + { + "label": "Evaluation pattern", + "reason": "This skill already carries eval assets, so benchmark how top examples define trigger boundaries and quality gates.", + } + ) + if (skill_dir / "scripts").exists(): + checks.append( + { + "label": "Execution pattern", + "reason": "There is deterministic logic in scripts, so compare how strong references separate prose from executable steps.", + } + ) + if (skill_dir / "agents" / "interface.yaml").exists(): + checks.append( + { + "label": "Portability pattern", + "reason": "The package carries neutral metadata, so scan how good references preserve semantics across targets without forking source.", + } + ) + checks.append( + { + "label": "Method pattern", + "reason": f"Use the core job description as the anchor for comparison: {description}", + } + ) + return checks[:4] + + +def build_summary(skill_dir: Path, references: list[dict]) -> dict: + skill_text = (skill_dir / "SKILL.md").read_text(encoding="utf-8") + frontmatter, body = parse_frontmatter(skill_text) + name = frontmatter.get("name", skill_dir.name) + description = frontmatter.get("description", "No description found.") + title = extract_title(body, name.replace("-", " ").title()) + focus = infer_scan_focus(skill_dir, description) + + borrow_plan = [ + "Method: borrow the smallest repeatable loop that improves reliability.", + "Structure: borrow directory or metadata patterns only when they reduce ambiguity.", + "Execution: borrow operator-facing flow, not source-specific ceremony.", + "Portability: borrow neutral metadata and degradation ideas, not client-specific lock-in.", + ] + + if references: + borrow_plan.append("Apply only the parts that fit the chosen archetype, gates, and context budget.") + + return { + "skill_name": name, + "title": title, + "description": description, + "scan_focus": focus, + "references": references, + "borrow_plan": borrow_plan, + "non_goals": [ + "Do not copy source prose or branding into the new skill.", + "Do not import gates that cost more context than they save.", + "Do not use benchmark scanning to justify scope creep.", + ], + } + + +def render_markdown(summary: dict) -> str: + lines = [ + "# Reference Scan", + "", + f"Skill: `{summary['skill_name']}`", + "", + "## Why This Step Exists", + "", + "Use a short benchmark pass before authoring the package in depth. The goal is to borrow durable patterns from strong reference objects without copying their prose or carrying their weight into the new skill.", + "", + "## Current Skill Anchor", + "", + f"- Title: `{summary['title']}`", + f"- Description: {summary['description']}", + "", + "## Scan Focus", + "", + ] + for item in summary["scan_focus"]: + lines.append(f"- **{item['label']}**: {item['reason']}") + + lines.extend(["", "## Reference Objects", ""]) + if summary["references"]: + for ref in summary["references"]: + lines.extend( + [ + f"### {ref['name']}", + f"- Category: `{ref['category']}`", + f"- Borrow: {ref['borrow']}", + f"- Avoid: {ref['avoid']}", + "", + ] + ) + else: + lines.extend( + [ + "- No explicit reference objects recorded yet.", + "- Recommended: capture 3 to 5 references at most.", + "- Suggested mix: one method reference, one structure reference, one execution or portability reference.", + "", + ] + ) + + lines.extend(["## Borrow Plan", ""]) + for item in summary["borrow_plan"]: + lines.append(f"- {item}") + + lines.extend(["", "## Non-Goals", ""]) + for item in summary["non_goals"]: + lines.append(f"- {item}") + + return "\n".join(lines).strip() + "\n" + + +def render_reference_scan(skill_dir: Path, references: list[dict], output_md: Path | None = None, output_json: Path | None = None) -> dict: + skill_dir = skill_dir.resolve() + reports_dir = skill_dir / "reports" + reports_dir.mkdir(parents=True, exist_ok=True) + output_md = output_md or reports_dir / "reference-scan.md" + output_json = output_json or reports_dir / "reference-scan.json" + + summary = build_summary(skill_dir, references) + output_md.write_text(render_markdown(summary), encoding="utf-8") + output_json.write_text(json.dumps(summary, ensure_ascii=False, indent=2), encoding="utf-8") + + return { + "ok": True, + "skill_dir": str(skill_dir), + "artifacts": { + "markdown": str(output_md), + "json": str(output_json), + }, + "summary": summary, + } + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a benchmark-oriented reference scan for a skill package.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--reference", action="append", default=[], help="Format: name::category::borrow::avoid") + parser.add_argument("--output-md") + parser.add_argument("--output-json") + args = parser.parse_args() + + refs = [parse_reference(item) for item in args.reference] + result = render_reference_scan( + Path(args.skill_dir), + refs, + Path(args.output_md).resolve() if args.output_md else None, + Path(args.output_json).resolve() if args.output_json else None, + ) + print(json.dumps(result, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/yao.py b/scripts/yao.py index 8bbbe5a..95624c5 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -225,6 +225,20 @@ def command_skill_report(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_reference_scan(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + for reference in args.reference: + cmd.extend(["--reference", reference]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + result = run_script("render_reference_scan.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_review(args: argparse.Namespace) -> int: target_name = resolve_promotion_target(args.target) bundle_dir = ROOT / "reports" / "iteration_bundles" / target_name @@ -434,6 +448,16 @@ def build_parser() -> argparse.ArgumentParser: skill_report_cmd.add_argument("--output-json") skill_report_cmd.set_defaults(func=command_skill_report) + reference_scan_cmd = subparsers.add_parser( + "reference-scan", + help="Render a controlled benchmark scan report for a skill package.", + ) + reference_scan_cmd.add_argument("skill_dir", nargs="?", default=".") + reference_scan_cmd.add_argument("--reference", action="append", default=[]) + reference_scan_cmd.add_argument("--output-md") + reference_scan_cmd.add_argument("--output-json") + reference_scan_cmd.set_defaults(func=command_reference_scan) + package_cmd = subparsers.add_parser("package", help="Export compatibility artifacts for selected targets.") package_cmd.add_argument("skill_dir", nargs="?", default=".") package_cmd.add_argument("--platform", action="append") diff --git a/tests/verify_reference_scan.py b/tests/verify_reference_scan.py new file mode 100644 index 0000000..4156e41 --- /dev/null +++ b/tests/verify_reference_scan.py @@ -0,0 +1,70 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_reference_scan.py" + + +def main() -> None: + tmp_root = ROOT / "tests" / "tmp_reference_scan" + if tmp_root.exists(): + subprocess.run(["rm", "-rf", str(tmp_root)], check=True) + tmp_root.mkdir(parents=True, exist_ok=True) + + init_proc = subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "init_skill.py"), + "reference-scan-demo", + "--description", + "Turn a product idea into a reusable benchmarked skill.", + "--output-dir", + str(tmp_root), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + init_payload = json.loads(init_proc.stdout) + created = Path(init_payload["root"]) + + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(created), + "--reference", + "Top Method::method::Borrow a tight benchmark loop.::Avoid copying source-specific branding.", + "--reference", + "Portable System::portability::Borrow neutral metadata and degradation rules.::Avoid target lock-in.", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + + md_path = Path(payload["artifacts"]["markdown"]) + json_path = Path(payload["artifacts"]["json"]) + assert md_path.exists(), md_path + assert json_path.exists(), json_path + + report_text = md_path.read_text(encoding="utf-8") + assert "# Reference Scan" in report_text, report_text[:200] + assert "Top Method" in report_text, report_text[:300] + assert "Borrow Plan" in report_text, report_text[:500] + + summary = json.loads(json_path.read_text(encoding="utf-8")) + assert len(summary["references"]) == 2, summary + + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index d39c84d..a61c7f2 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -37,6 +37,7 @@ def main() -> None: assert (created / "SKILL.md").exists(), created assert (created / "README.md").exists(), created assert (created / "reports" / "skill-overview.html").exists(), created + assert (created / "reports" / "reference-scan.md").exists(), created validate_result = run("validate", str(created)) assert validate_result["ok"], validate_result @@ -46,6 +47,15 @@ def main() -> None: assert skill_report_result["ok"], skill_report_result assert skill_report_result["payload"]["artifacts"]["html"].endswith("reports/skill-overview.html"), skill_report_result + reference_scan_result = run( + "reference-scan", + str(created), + "--reference", + "World Class Method::method::Borrow the smallest repeatable evaluation loop.::Do not copy heavy ceremony.", + ) + assert reference_scan_result["ok"], reference_scan_result + assert reference_scan_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-scan.md"), reference_scan_result + optimize_result = run("optimize-description", "--target", "root") assert optimize_result["ok"], optimize_result assert optimize_result["payload"]["winner"]["label"] == "Current", optimize_result