#!/usr/bin/env python3 import json import os import subprocess import sys from pathlib import Path ROOT = Path(__file__).resolve().parent.parent CLI = ROOT / "scripts" / "yao.py" BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan" def run(*args: str, input_text: str | None = None) -> dict: proc = subprocess.run( [sys.executable, str(CLI), *args], cwd=ROOT, capture_output=True, text=True, input=input_text, env=os.environ, ) payload = json.loads(proc.stdout) return { "ok": proc.returncode == 0, "returncode": proc.returncode, "payload": payload, "stderr": proc.stderr, } def main() -> None: tmp_root = ROOT / "tests" / "tmp_cli" if tmp_root.exists(): subprocess.run(["rm", "-rf", str(tmp_root)], check=True) tmp_root.mkdir(parents=True, exist_ok=True) remote_version = tmp_root / "remote-version.txt" remote_version.write_text("9.9.9\n", encoding="utf-8") init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root)) assert init_result["ok"], init_result created = Path(init_result["payload"]["root"]) assert (created / "SKILL.md").exists(), created assert (created / "README.md").exists(), created assert (created / "reports" / "intent-dialogue.md").exists(), created assert (created / "reports" / "intent-confidence.md").exists(), created assert (created / "reports" / "skill-overview.html").exists(), created assert (created / "reports" / "review-studio.html").exists(), created assert (created / "reports" / "review-studio.json").exists(), created assert (created / "reports" / "review-viewer.html").exists(), created assert (created / "reports" / "reference-scan.md").exists(), created assert (created / "reports" / "reference-synthesis.md").exists(), created assert (created / "reports" / "output-risk-profile.md").exists(), created assert (created / "reports" / "artifact-design-profile.md").exists(), created assert (created / "reports" / "prompt-quality-profile.md").exists(), created assert (created / "reports" / "system-model.md").exists(), created assert (created / "reports" / "skill-ir.json").exists(), created assert (created / "reports" / "iteration-directions.md").exists(), created assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created init_report_view = init_result["payload"]["report_view"] assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view assert Path(init_report_view["html_report"]).exists(), init_report_view assert init_report_view["review_studio"].endswith("reports/review-studio.html"), init_report_view assert Path(init_report_view["review_studio"]).exists(), init_report_view assert "Skill 已创建完成" in init_report_view["message"], init_report_view assert "Review Studio 2.0" in init_report_view["message"], init_report_view assert "概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view assert "默认使用中文简体" in init_report_view["message"], init_report_view assert "切换英文版" in init_report_view["message"], init_report_view init_skill_ir = init_result["payload"]["skill_ir"] assert init_skill_ir["name"] == "cli-demo-skill", init_skill_ir assert init_skill_ir["trigger_samples"] >= 1, init_skill_ir quickstart_result = run( "quickstart", "--output-dir", str(tmp_root), "--github-fixture-dir", str(BENCHMARK_FIXTURE_DIR), "--no-update-check", input_text=( "quickstart-skill\n" "Turn messy release notes into a reusable release brief skill.\n" "release notes, changelog snippets\n" "A reusable markdown release brief.\n" "looks right\n" "It should not publish blog posts or send email.\n" "consistency, portability\n" "production\n" "production\n" "\n" "privacy and naming\n" ), ) assert quickstart_result["ok"], quickstart_result quickstart_root = Path(quickstart_result["payload"]["root"]) assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root assert (quickstart_root / "reports" / "review-studio.html").exists(), quickstart_root assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root assert (quickstart_root / "reports" / "intent-confidence.md").exists(), quickstart_root assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root assert (quickstart_root / "reports" / "artifact-design-profile.md").exists(), quickstart_root assert (quickstart_root / "reports" / "prompt-quality-profile.md").exists(), quickstart_root assert (quickstart_root / "reports" / "system-model.md").exists(), quickstart_root assert quickstart_result["payload"]["archetype"] == "production", quickstart_result assert quickstart_result["payload"]["guidance"]["experience_note"], quickstart_result assert quickstart_result["payload"]["guidance"]["problem_diagnosis"]["candidates"], quickstart_result assert quickstart_result["payload"]["intent_confidence"]["score"] >= 70, quickstart_result assert quickstart_result["payload"]["recommendation"]["summary"], quickstart_result assert quickstart_result["payload"]["reference_mode"]["mode"] == "silent", quickstart_result quickstart_report_view = quickstart_result["payload"]["report_view"] assert quickstart_report_view["html_report"].endswith("reports/skill-overview.html"), quickstart_report_view assert Path(quickstart_report_view["html_report"]).exists(), quickstart_report_view assert Path(quickstart_report_view["review_studio"]).exists(), quickstart_report_view assert "Skill 已创建完成" in quickstart_report_view["message"], quickstart_report_view assert "默认使用中文简体" in quickstart_report_view["message"], quickstart_report_view assert quickstart_result["payload"]["guidance"]["next_steps"][0].startswith("Open reports/skill-overview.html"), quickstart_result assert "reports/review-studio.html" in quickstart_result["payload"]["guidance"]["next_steps"][2], quickstart_result assert "audit report" in quickstart_result["payload"]["guidance"]["next_steps"][0], quickstart_result assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["reference_synthesis"].endswith( "reports/reference-synthesis.md" ), quickstart_result assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["prompt_quality_profile"].endswith( "reports/prompt-quality-profile.md" ), quickstart_result assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["system_model"].endswith( "reports/system-model.md" ), quickstart_result assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["review_studio"].endswith( "reports/review-studio.html" ), quickstart_result assert "uncertainty_or_conflict" not in quickstart_result["payload"], quickstart_result quickstart_manifest = json.loads((quickstart_root / "manifest.json").read_text(encoding="utf-8")) assert quickstart_manifest["status"] == "active", quickstart_manifest assert quickstart_manifest["lifecycle_stage"] == "production", quickstart_manifest quickstart_validate_result = run("validate", str(quickstart_root), "--require-manifest") assert quickstart_validate_result["ok"], quickstart_validate_result quickstart_conflict_result = run( "quickstart", "--output-dir", str(tmp_root), "--github-fixture-dir", str(BENCHMARK_FIXTURE_DIR), "--no-update-check", input_text=( "quickstart-conflict-skill\n" "Turn repeated release notes into a governed release command skill.\n" "release notes, changelog snippets\n" "A governed release packet.\n" "looks right\n" "It should not publish blog posts or send email.\n" "auditability, portability\n" "governed\n" "governed\n" "Minimal vibe helper::taste::Keep the first pass fast, minimal, and lightweight.::Do not add review, governance, or approval steps.\n" "privacy and naming\n" ), ) assert quickstart_conflict_result["ok"], quickstart_conflict_result assert quickstart_conflict_result["payload"]["reference_mode"]["mode"] == "explicit", quickstart_conflict_result assert quickstart_conflict_result["payload"]["uncertainty_or_conflict"]["conflicts"], quickstart_conflict_result validate_result = run("validate", str(created)) assert validate_result["ok"], validate_result assert len(validate_result["payload"]["steps"]) == 4, validate_result skill_report_result = run("skill-report", str(created)) assert skill_report_result["ok"], skill_report_result assert skill_report_result["payload"]["artifacts"]["html"].endswith("reports/skill-overview.html"), skill_report_result review_viewer_result = run("review-viewer", str(created)) assert review_viewer_result["ok"], review_viewer_result assert review_viewer_result["payload"]["artifacts"]["html"].endswith("reports/review-viewer.html"), review_viewer_result review_studio_result = run("review-studio", str(created)) assert review_studio_result["ok"], review_studio_result assert review_studio_result["payload"]["artifacts"]["html"].endswith("reports/review-studio.html"), review_studio_result assert review_studio_result["payload"]["summary"]["gate_count"] == 8, review_studio_result reference_scan_result = run( "reference-scan", str(created), "--external-reference", "World Class Method::method::Borrow the smallest repeatable evaluation loop.::Do not copy heavy ceremony.", "--user-reference", "Product I Admire::taste::Learn the calm structure and clarity of output.::Do not copy wording.", "--local-constraint", "Local Naming::structure::Keep folder naming aligned with the local library.::Do not inherit private references.", ) assert reference_scan_result["ok"], reference_scan_result assert reference_scan_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-scan.md"), reference_scan_result assert len(reference_scan_result["payload"]["summary"]["user_references"]) == 1, reference_scan_result github_benchmark_result = run( "github-benchmark-scan", str(created), "--query", "workflow evaluation portability", "--fixture-dir", str(BENCHMARK_FIXTURE_DIR), ) assert github_benchmark_result["ok"], github_benchmark_result assert len(github_benchmark_result["payload"]["repositories"]) == 3, github_benchmark_result intent_confidence_result = run("intent-confidence", str(created)) assert intent_confidence_result["ok"], intent_confidence_result assert intent_confidence_result["payload"]["summary"]["score"] >= 0, intent_confidence_result intent_result = run("intent-dialogue", str(created)) assert intent_result["ok"], intent_result assert intent_result["payload"]["artifacts"]["markdown"].endswith("reports/intent-dialogue.md"), intent_result reference_synthesis_result = run("reference-synthesis", str(created)) assert reference_synthesis_result["ok"], reference_synthesis_result assert reference_synthesis_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-synthesis.md"), reference_synthesis_result output_risk_result = run("output-risk-profile", str(created)) assert output_risk_result["ok"], output_risk_result assert output_risk_result["payload"]["artifacts"]["markdown"].endswith("reports/output-risk-profile.md"), output_risk_result assert output_risk_result["payload"]["summary"]["risk_families"], output_risk_result artifact_design_result = run("artifact-design-profile", str(created)) assert artifact_design_result["ok"], artifact_design_result assert artifact_design_result["payload"]["artifacts"]["markdown"].endswith("reports/artifact-design-profile.md"), artifact_design_result assert artifact_design_result["payload"]["summary"]["quality_gates"], artifact_design_result prompt_quality_result = run("prompt-quality-profile", str(created)) assert prompt_quality_result["ok"], prompt_quality_result assert prompt_quality_result["payload"]["artifacts"]["markdown"].endswith("reports/prompt-quality-profile.md"), prompt_quality_result assert prompt_quality_result["payload"]["summary"]["quality_matrix"], prompt_quality_result system_model_result = run("system-model", str(created)) assert system_model_result["ok"], system_model_result assert system_model_result["payload"]["artifacts"]["markdown"].endswith("reports/system-model.md"), system_model_result assert system_model_result["payload"]["summary"]["stability"]["score"] >= 0, system_model_result directions_result = run("iteration-directions", str(created)) assert directions_result["ok"], directions_result assert directions_result["payload"]["artifacts"]["markdown"].endswith("reports/iteration-directions.md"), directions_result skill_ir_result = run("skill-ir", str(created)) assert skill_ir_result["ok"], skill_ir_result assert skill_ir_result["payload"]["artifacts"]["json"].endswith("reports/skill-ir.json"), skill_ir_result created_skill_ir = json.loads((created / "reports" / "skill-ir.json").read_text(encoding="utf-8")) assert created_skill_ir["schema_version"] == "2.0.0", created_skill_ir assert created_skill_ir["trigger_surface"]["description"], created_skill_ir output_eval_result = run( "output-eval", "--cases", str(ROOT / "evals" / "output" / "cases.jsonl"), "--output-json", str(created / "reports" / "output_quality_scorecard.json"), "--output-md", str(created / "reports" / "output_quality_scorecard.md"), ) assert output_eval_result["ok"], output_eval_result assert output_eval_result["payload"]["summary"]["with_skill_pass_rate"] > output_eval_result["payload"]["summary"]["baseline_pass_rate"], output_eval_result conformance_result = run("conformance", str(created)) assert conformance_result["ok"], conformance_result assert conformance_result["payload"]["summary"]["target_count"] == 5, conformance_result assert conformance_result["payload"]["artifacts"]["markdown"].endswith("reports/conformance_matrix.md"), conformance_result trust_result = run("trust", str(created)) assert trust_result["ok"], trust_result assert trust_result["payload"]["summary"]["secret_findings"] == 0, trust_result assert trust_result["payload"]["artifacts"]["markdown"].endswith("reports/security_trust_report.md"), trust_result atlas_result = run( "skill-atlas", "--workspace-root", str(tmp_root), "--output-dir", str(tmp_root / "skill_atlas"), "--report-html", str(tmp_root / "skill_atlas.html"), "--report-json", str(tmp_root / "skill_atlas.json"), "--today", "2026-06-13", ) assert atlas_result["ok"], atlas_result assert atlas_result["payload"]["summary"]["skill_count"] >= 2, atlas_result assert atlas_result["payload"]["artifacts"]["report_html"].endswith("skill_atlas.html"), atlas_result feedback_result = run( "feedback", str(created), "--note", "Keep the first version light and tighten exclusions before adding scripts.", "--rating", "4", "--category", "boundary", "--recommended-action", "tighten-trigger", ) assert feedback_result["ok"], feedback_result assert feedback_result["payload"]["feedback"]["summary"]["count"] == 1, feedback_result optimize_result = run("optimize-description", "--target", "root") assert optimize_result["ok"], optimize_result assert optimize_result["payload"]["winner"]["label"] == "Current", optimize_result baseline_compare_result = run("baseline-compare") assert baseline_compare_result["ok"], baseline_compare_result assert baseline_compare_result["payload"]["summary"]["target_count"] == 3, baseline_compare_result promote_result = run("promote-check") assert promote_result["ok"], promote_result assert promote_result["payload"]["summary"]["blocked"] == 0, promote_result review_result = run("review", "--target", "root") assert review_result["ok"], review_result assert review_result["payload"]["artifacts"]["review_md"].endswith("reports/iteration_bundles/yao-meta-skill/review.md") snapshot_result = run("release-snapshot", "--target", "root", "--label", "cli-smoke") assert snapshot_result["ok"], snapshot_result assert snapshot_result["payload"]["artifacts"]["snapshot_json"].endswith("cli-smoke.json"), snapshot_result flow_result = run("workspace-flow", "--target", "root", "--label", "cli-flow") assert flow_result["ok"], flow_result assert flow_result["payload"]["artifacts"][0]["snapshot"]["artifacts"]["snapshot_md"].endswith("cli-flow.md"), flow_result report_result = run("report") assert report_result["ok"], report_result assert "iteration_ledger" in report_result["payload"]["artifacts"], report_result assert "portability_score" in report_result["payload"]["artifacts"], report_result assert "artifact_design_profile" in report_result["payload"]["artifacts"], report_result assert "prompt_quality_profile" in report_result["payload"]["artifacts"], report_result package_dir = tmp_root / "dist" package_result = run("package", ".", "--platform", "generic", "--output-dir", str(package_dir)) assert package_result["ok"], package_result assert (package_dir / "targets" / "generic" / "adapter.json").exists(), package_dir update_result = run( "check-update", "--force", "--no-cache", "--version-url", remote_version.as_uri(), "--manifest-url", remote_version.as_uri(), ) assert not update_result["ok"], update_result assert update_result["returncode"] == 2, update_result assert "Update URL scheme is not allowed: file" in update_result["payload"]["error"], update_result test_result = run("test", "--target", "promotion-check") assert test_result["ok"], test_result print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) if __name__ == "__main__": main()