yaojingang
360 lines
18 KiB
Python
360 lines
18 KiB
Python
#!/usr/bin/env python3
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
CLI = ROOT / "scripts" / "yao.py"
|
|
BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan"
|
|
|
|
|
|
def run(*args: str, input_text: str | None = None) -> dict:
|
|
proc = subprocess.run(
|
|
[sys.executable, str(CLI), *args],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
input=input_text,
|
|
env=os.environ,
|
|
)
|
|
payload = json.loads(proc.stdout)
|
|
return {
|
|
"ok": proc.returncode == 0,
|
|
"returncode": proc.returncode,
|
|
"payload": payload,
|
|
"stderr": proc.stderr,
|
|
}
|
|
|
|
|
|
def main() -> None:
|
|
tmp_root = ROOT / "tests" / "tmp_cli"
|
|
if tmp_root.exists():
|
|
subprocess.run(["rm", "-rf", str(tmp_root)], check=True)
|
|
tmp_root.mkdir(parents=True, exist_ok=True)
|
|
remote_version = tmp_root / "remote-version.txt"
|
|
remote_version.write_text("9.9.9\n", encoding="utf-8")
|
|
|
|
init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root))
|
|
assert init_result["ok"], init_result
|
|
created = Path(init_result["payload"]["root"])
|
|
assert (created / "SKILL.md").exists(), created
|
|
assert (created / "README.md").exists(), created
|
|
assert (created / "reports" / "intent-dialogue.md").exists(), created
|
|
assert (created / "reports" / "intent-confidence.md").exists(), created
|
|
assert (created / "reports" / "skill-overview.html").exists(), created
|
|
assert (created / "reports" / "review-studio.html").exists(), created
|
|
assert (created / "reports" / "review-studio.json").exists(), created
|
|
assert (created / "reports" / "review-viewer.html").exists(), created
|
|
assert (created / "reports" / "reference-scan.md").exists(), created
|
|
assert (created / "reports" / "reference-synthesis.md").exists(), created
|
|
assert (created / "reports" / "output-risk-profile.md").exists(), created
|
|
assert (created / "reports" / "artifact-design-profile.md").exists(), created
|
|
assert (created / "reports" / "prompt-quality-profile.md").exists(), created
|
|
assert (created / "reports" / "system-model.md").exists(), created
|
|
assert (created / "reports" / "skill-ir.json").exists(), created
|
|
assert (created / "reports" / "iteration-directions.md").exists(), created
|
|
assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created
|
|
init_report_view = init_result["payload"]["report_view"]
|
|
assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view
|
|
assert Path(init_report_view["html_report"]).exists(), init_report_view
|
|
assert init_report_view["review_studio"].endswith("reports/review-studio.html"), init_report_view
|
|
assert Path(init_report_view["review_studio"]).exists(), init_report_view
|
|
assert "Skill 已创建完成" in init_report_view["message"], init_report_view
|
|
assert "Review Studio 2.0" in init_report_view["message"], init_report_view
|
|
assert "概述、指标、原理、触发边界、输入输出、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view
|
|
assert "默认使用中文简体" in init_report_view["message"], init_report_view
|
|
assert "切换英文版" in init_report_view["message"], init_report_view
|
|
init_skill_ir = init_result["payload"]["skill_ir"]
|
|
assert init_skill_ir["name"] == "cli-demo-skill", init_skill_ir
|
|
assert init_skill_ir["trigger_samples"] >= 1, init_skill_ir
|
|
|
|
quickstart_result = run(
|
|
"quickstart",
|
|
"--output-dir",
|
|
str(tmp_root),
|
|
"--github-fixture-dir",
|
|
str(BENCHMARK_FIXTURE_DIR),
|
|
"--no-update-check",
|
|
input_text=(
|
|
"quickstart-skill\n"
|
|
"Turn messy release notes into a reusable release brief skill.\n"
|
|
"release notes, changelog snippets\n"
|
|
"A reusable markdown release brief.\n"
|
|
"looks right\n"
|
|
"It should not publish blog posts or send email.\n"
|
|
"consistency, portability\n"
|
|
"production\n"
|
|
"production\n"
|
|
"\n"
|
|
"privacy and naming\n"
|
|
),
|
|
)
|
|
assert quickstart_result["ok"], quickstart_result
|
|
quickstart_root = Path(quickstart_result["payload"]["root"])
|
|
assert (quickstart_root / "reports" / "review-viewer.html").exists(), quickstart_root
|
|
assert (quickstart_root / "reports" / "review-studio.html").exists(), quickstart_root
|
|
assert (quickstart_root / "reports" / "github-benchmark-scan.md").exists(), quickstart_root
|
|
assert (quickstart_root / "reports" / "intent-confidence.md").exists(), quickstart_root
|
|
assert (quickstart_root / "reports" / "reference-synthesis.md").exists(), quickstart_root
|
|
assert (quickstart_root / "reports" / "artifact-design-profile.md").exists(), quickstart_root
|
|
assert (quickstart_root / "reports" / "prompt-quality-profile.md").exists(), quickstart_root
|
|
assert (quickstart_root / "reports" / "system-model.md").exists(), quickstart_root
|
|
assert quickstart_result["payload"]["archetype"] == "production", quickstart_result
|
|
assert quickstart_result["payload"]["guidance"]["experience_note"], quickstart_result
|
|
assert quickstart_result["payload"]["guidance"]["problem_diagnosis"]["candidates"], quickstart_result
|
|
assert quickstart_result["payload"]["intent_confidence"]["score"] >= 70, quickstart_result
|
|
assert quickstart_result["payload"]["recommendation"]["summary"], quickstart_result
|
|
assert quickstart_result["payload"]["reference_mode"]["mode"] == "silent", quickstart_result
|
|
quickstart_report_view = quickstart_result["payload"]["report_view"]
|
|
assert quickstart_report_view["html_report"].endswith("reports/skill-overview.html"), quickstart_report_view
|
|
assert Path(quickstart_report_view["html_report"]).exists(), quickstart_report_view
|
|
assert Path(quickstart_report_view["review_studio"]).exists(), quickstart_report_view
|
|
assert "Skill 已创建完成" in quickstart_report_view["message"], quickstart_report_view
|
|
assert "默认使用中文简体" in quickstart_report_view["message"], quickstart_report_view
|
|
assert quickstart_result["payload"]["guidance"]["next_steps"][0].startswith("Open reports/skill-overview.html"), quickstart_result
|
|
assert "reports/review-studio.html" in quickstart_result["payload"]["guidance"]["next_steps"][2], quickstart_result
|
|
assert "audit report" in quickstart_result["payload"]["guidance"]["next_steps"][0], quickstart_result
|
|
assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["reference_synthesis"].endswith(
|
|
"reports/reference-synthesis.md"
|
|
), quickstart_result
|
|
assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["prompt_quality_profile"].endswith(
|
|
"reports/prompt-quality-profile.md"
|
|
), quickstart_result
|
|
assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["system_model"].endswith(
|
|
"reports/system-model.md"
|
|
), quickstart_result
|
|
assert quickstart_result["payload"]["reviewer_evidence"]["artifacts"]["review_studio"].endswith(
|
|
"reports/review-studio.html"
|
|
), quickstart_result
|
|
assert "uncertainty_or_conflict" not in quickstart_result["payload"], quickstart_result
|
|
quickstart_manifest = json.loads((quickstart_root / "manifest.json").read_text(encoding="utf-8"))
|
|
assert quickstart_manifest["status"] == "active", quickstart_manifest
|
|
assert quickstart_manifest["lifecycle_stage"] == "production", quickstart_manifest
|
|
quickstart_validate_result = run("validate", str(quickstart_root), "--require-manifest")
|
|
assert quickstart_validate_result["ok"], quickstart_validate_result
|
|
|
|
quickstart_conflict_result = run(
|
|
"quickstart",
|
|
"--output-dir",
|
|
str(tmp_root),
|
|
"--github-fixture-dir",
|
|
str(BENCHMARK_FIXTURE_DIR),
|
|
"--no-update-check",
|
|
input_text=(
|
|
"quickstart-conflict-skill\n"
|
|
"Turn repeated release notes into a governed release command skill.\n"
|
|
"release notes, changelog snippets\n"
|
|
"A governed release packet.\n"
|
|
"looks right\n"
|
|
"It should not publish blog posts or send email.\n"
|
|
"auditability, portability\n"
|
|
"governed\n"
|
|
"governed\n"
|
|
"Minimal vibe helper::taste::Keep the first pass fast, minimal, and lightweight.::Do not add review, governance, or approval steps.\n"
|
|
"privacy and naming\n"
|
|
),
|
|
)
|
|
assert quickstart_conflict_result["ok"], quickstart_conflict_result
|
|
assert quickstart_conflict_result["payload"]["reference_mode"]["mode"] == "explicit", quickstart_conflict_result
|
|
assert quickstart_conflict_result["payload"]["uncertainty_or_conflict"]["conflicts"], quickstart_conflict_result
|
|
|
|
validate_result = run("validate", str(created))
|
|
assert validate_result["ok"], validate_result
|
|
assert len(validate_result["payload"]["steps"]) == 4, validate_result
|
|
|
|
skill_report_result = run("skill-report", str(created))
|
|
assert skill_report_result["ok"], skill_report_result
|
|
assert skill_report_result["payload"]["artifacts"]["html"].endswith("reports/skill-overview.html"), skill_report_result
|
|
|
|
review_viewer_result = run("review-viewer", str(created))
|
|
assert review_viewer_result["ok"], review_viewer_result
|
|
assert review_viewer_result["payload"]["artifacts"]["html"].endswith("reports/review-viewer.html"), review_viewer_result
|
|
|
|
review_studio_result = run("review-studio", str(created))
|
|
assert review_studio_result["ok"], review_studio_result
|
|
assert review_studio_result["payload"]["artifacts"]["html"].endswith("reports/review-studio.html"), review_studio_result
|
|
assert review_studio_result["payload"]["summary"]["gate_count"] == 8, review_studio_result
|
|
|
|
reference_scan_result = run(
|
|
"reference-scan",
|
|
str(created),
|
|
"--external-reference",
|
|
"World Class Method::method::Borrow the smallest repeatable evaluation loop.::Do not copy heavy ceremony.",
|
|
"--user-reference",
|
|
"Product I Admire::taste::Learn the calm structure and clarity of output.::Do not copy wording.",
|
|
"--local-constraint",
|
|
"Local Naming::structure::Keep folder naming aligned with the local library.::Do not inherit private references.",
|
|
)
|
|
assert reference_scan_result["ok"], reference_scan_result
|
|
assert reference_scan_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-scan.md"), reference_scan_result
|
|
assert len(reference_scan_result["payload"]["summary"]["user_references"]) == 1, reference_scan_result
|
|
|
|
github_benchmark_result = run(
|
|
"github-benchmark-scan",
|
|
str(created),
|
|
"--query",
|
|
"workflow evaluation portability",
|
|
"--fixture-dir",
|
|
str(BENCHMARK_FIXTURE_DIR),
|
|
)
|
|
assert github_benchmark_result["ok"], github_benchmark_result
|
|
assert len(github_benchmark_result["payload"]["repositories"]) == 3, github_benchmark_result
|
|
|
|
intent_confidence_result = run("intent-confidence", str(created))
|
|
assert intent_confidence_result["ok"], intent_confidence_result
|
|
assert intent_confidence_result["payload"]["summary"]["score"] >= 0, intent_confidence_result
|
|
|
|
intent_result = run("intent-dialogue", str(created))
|
|
assert intent_result["ok"], intent_result
|
|
assert intent_result["payload"]["artifacts"]["markdown"].endswith("reports/intent-dialogue.md"), intent_result
|
|
|
|
reference_synthesis_result = run("reference-synthesis", str(created))
|
|
assert reference_synthesis_result["ok"], reference_synthesis_result
|
|
assert reference_synthesis_result["payload"]["artifacts"]["markdown"].endswith("reports/reference-synthesis.md"), reference_synthesis_result
|
|
|
|
output_risk_result = run("output-risk-profile", str(created))
|
|
assert output_risk_result["ok"], output_risk_result
|
|
assert output_risk_result["payload"]["artifacts"]["markdown"].endswith("reports/output-risk-profile.md"), output_risk_result
|
|
assert output_risk_result["payload"]["summary"]["risk_families"], output_risk_result
|
|
|
|
artifact_design_result = run("artifact-design-profile", str(created))
|
|
assert artifact_design_result["ok"], artifact_design_result
|
|
assert artifact_design_result["payload"]["artifacts"]["markdown"].endswith("reports/artifact-design-profile.md"), artifact_design_result
|
|
assert artifact_design_result["payload"]["summary"]["quality_gates"], artifact_design_result
|
|
|
|
prompt_quality_result = run("prompt-quality-profile", str(created))
|
|
assert prompt_quality_result["ok"], prompt_quality_result
|
|
assert prompt_quality_result["payload"]["artifacts"]["markdown"].endswith("reports/prompt-quality-profile.md"), prompt_quality_result
|
|
assert prompt_quality_result["payload"]["summary"]["quality_matrix"], prompt_quality_result
|
|
|
|
system_model_result = run("system-model", str(created))
|
|
assert system_model_result["ok"], system_model_result
|
|
assert system_model_result["payload"]["artifacts"]["markdown"].endswith("reports/system-model.md"), system_model_result
|
|
assert system_model_result["payload"]["summary"]["stability"]["score"] >= 0, system_model_result
|
|
|
|
directions_result = run("iteration-directions", str(created))
|
|
assert directions_result["ok"], directions_result
|
|
assert directions_result["payload"]["artifacts"]["markdown"].endswith("reports/iteration-directions.md"), directions_result
|
|
|
|
skill_ir_result = run("skill-ir", str(created))
|
|
assert skill_ir_result["ok"], skill_ir_result
|
|
assert skill_ir_result["payload"]["artifacts"]["json"].endswith("reports/skill-ir.json"), skill_ir_result
|
|
created_skill_ir = json.loads((created / "reports" / "skill-ir.json").read_text(encoding="utf-8"))
|
|
assert created_skill_ir["schema_version"] == "2.0.0", created_skill_ir
|
|
assert created_skill_ir["trigger_surface"]["description"], created_skill_ir
|
|
|
|
output_eval_result = run(
|
|
"output-eval",
|
|
"--cases",
|
|
str(ROOT / "evals" / "output" / "cases.jsonl"),
|
|
"--output-json",
|
|
str(created / "reports" / "output_quality_scorecard.json"),
|
|
"--output-md",
|
|
str(created / "reports" / "output_quality_scorecard.md"),
|
|
)
|
|
assert output_eval_result["ok"], output_eval_result
|
|
assert output_eval_result["payload"]["summary"]["with_skill_pass_rate"] > output_eval_result["payload"]["summary"]["baseline_pass_rate"], output_eval_result
|
|
|
|
conformance_result = run("conformance", str(created))
|
|
assert conformance_result["ok"], conformance_result
|
|
assert conformance_result["payload"]["summary"]["target_count"] == 5, conformance_result
|
|
assert conformance_result["payload"]["artifacts"]["markdown"].endswith("reports/conformance_matrix.md"), conformance_result
|
|
|
|
trust_result = run("trust", str(created))
|
|
assert trust_result["ok"], trust_result
|
|
assert trust_result["payload"]["summary"]["secret_findings"] == 0, trust_result
|
|
assert trust_result["payload"]["artifacts"]["markdown"].endswith("reports/security_trust_report.md"), trust_result
|
|
|
|
atlas_result = run(
|
|
"skill-atlas",
|
|
"--workspace-root",
|
|
str(tmp_root),
|
|
"--output-dir",
|
|
str(tmp_root / "skill_atlas"),
|
|
"--report-html",
|
|
str(tmp_root / "skill_atlas.html"),
|
|
"--report-json",
|
|
str(tmp_root / "skill_atlas.json"),
|
|
"--today",
|
|
"2026-06-13",
|
|
)
|
|
assert atlas_result["ok"], atlas_result
|
|
assert atlas_result["payload"]["summary"]["skill_count"] >= 2, atlas_result
|
|
assert atlas_result["payload"]["artifacts"]["report_html"].endswith("skill_atlas.html"), atlas_result
|
|
|
|
feedback_result = run(
|
|
"feedback",
|
|
str(created),
|
|
"--note",
|
|
"Keep the first version light and tighten exclusions before adding scripts.",
|
|
"--rating",
|
|
"4",
|
|
"--category",
|
|
"boundary",
|
|
"--recommended-action",
|
|
"tighten-trigger",
|
|
)
|
|
assert feedback_result["ok"], feedback_result
|
|
assert feedback_result["payload"]["feedback"]["summary"]["count"] == 1, feedback_result
|
|
|
|
optimize_result = run("optimize-description", "--target", "root")
|
|
assert optimize_result["ok"], optimize_result
|
|
assert optimize_result["payload"]["winner"]["label"] == "Current", optimize_result
|
|
|
|
baseline_compare_result = run("baseline-compare")
|
|
assert baseline_compare_result["ok"], baseline_compare_result
|
|
assert baseline_compare_result["payload"]["summary"]["target_count"] == 3, baseline_compare_result
|
|
|
|
promote_result = run("promote-check")
|
|
assert promote_result["ok"], promote_result
|
|
assert promote_result["payload"]["summary"]["blocked"] == 0, promote_result
|
|
|
|
review_result = run("review", "--target", "root")
|
|
assert review_result["ok"], review_result
|
|
assert review_result["payload"]["artifacts"]["review_md"].endswith("reports/iteration_bundles/yao-meta-skill/review.md")
|
|
|
|
snapshot_result = run("release-snapshot", "--target", "root", "--label", "cli-smoke")
|
|
assert snapshot_result["ok"], snapshot_result
|
|
assert snapshot_result["payload"]["artifacts"]["snapshot_json"].endswith("cli-smoke.json"), snapshot_result
|
|
|
|
flow_result = run("workspace-flow", "--target", "root", "--label", "cli-flow")
|
|
assert flow_result["ok"], flow_result
|
|
assert flow_result["payload"]["artifacts"][0]["snapshot"]["artifacts"]["snapshot_md"].endswith("cli-flow.md"), flow_result
|
|
|
|
report_result = run("report")
|
|
assert report_result["ok"], report_result
|
|
assert "iteration_ledger" in report_result["payload"]["artifacts"], report_result
|
|
assert "portability_score" in report_result["payload"]["artifacts"], report_result
|
|
assert "artifact_design_profile" in report_result["payload"]["artifacts"], report_result
|
|
assert "prompt_quality_profile" in report_result["payload"]["artifacts"], report_result
|
|
|
|
package_dir = tmp_root / "dist"
|
|
package_result = run("package", ".", "--platform", "generic", "--output-dir", str(package_dir))
|
|
assert package_result["ok"], package_result
|
|
assert (package_dir / "targets" / "generic" / "adapter.json").exists(), package_dir
|
|
|
|
update_result = run(
|
|
"check-update",
|
|
"--force",
|
|
"--no-cache",
|
|
"--version-url",
|
|
remote_version.as_uri(),
|
|
"--manifest-url",
|
|
remote_version.as_uri(),
|
|
)
|
|
assert not update_result["ok"], update_result
|
|
assert update_result["returncode"] == 2, update_result
|
|
assert "Update URL scheme is not allowed: file" in update_result["payload"]["error"], update_result
|
|
|
|
test_result = run("test", "--target", "promotion-check")
|
|
assert test_result["ok"], test_result
|
|
|
|
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|