From e3583647c74d6701f34afdb5abdb2aec3daaa518 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Mon, 15 Jun 2026 22:32:57 +0800 Subject: [PATCH] Refactor Yao CLI command domains --- .../render_architecture_maintainability.py | 28 +- scripts/review_studio_data.py | 3 +- scripts/review_studio_gates.py | 3 +- scripts/yao.py | 382 +----------------- scripts/yao_cli_distribution_commands.py | 164 ++++++++ scripts/yao_cli_output_commands.py | 106 +++++ scripts/yao_cli_report_commands.py | 96 ++++- tests/verify_architecture_maintainability.py | 4 +- tests/verify_trust_check.py | 8 + tests/verify_yao_cli.py | 11 +- 10 files changed, 426 insertions(+), 379 deletions(-) create mode 100644 scripts/yao_cli_distribution_commands.py create mode 100644 scripts/yao_cli_output_commands.py diff --git a/scripts/render_architecture_maintainability.py b/scripts/render_architecture_maintainability.py index c39804d3..d4d32de5 100644 --- a/scripts/render_architecture_maintainability.py +++ b/scripts/render_architecture_maintainability.py @@ -66,15 +66,23 @@ def recommendation_for(path: str) -> str: return "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." -def count_command_handlers(skill_dir: Path) -> int: - path = skill_dir / "scripts" / "yao.py" +def count_handlers_in_file(path: Path) -> int: if not path.exists(): return 0 - count = 0 - for line in path.read_text(encoding="utf-8", errors="replace").splitlines(): - if line.startswith("def command_"): - count += 1 - return count + return sum(1 for line in path.read_text(encoding="utf-8", errors="replace").splitlines() if line.startswith("def command_")) + + +def command_module_paths(skill_dir: Path) -> list[Path]: + scripts_dir = skill_dir / "scripts" + if not scripts_dir.exists(): + return [] + paths = [scripts_dir / "yao.py"] + paths.extend(sorted(scripts_dir.glob("yao_cli_*commands.py"))) + return [path for path in paths if path.exists()] + + +def count_cli_command_handlers(skill_dir: Path) -> int: + return sum(count_handlers_in_file(path) for path in command_module_paths(skill_dir)) def build_report(skill_dir: Path, warn_lines: int, block_lines: int, generated_at: str) -> dict[str, Any]: @@ -120,7 +128,9 @@ def build_report(skill_dir: Path, warn_lines: int, block_lines: int, generated_a "test_file_count": test_count, "internal_module_count": internal_count, "cli_script_count": cli_count, - "command_handler_count": count_command_handlers(skill_dir), + "command_handler_count": count_cli_command_handlers(skill_dir), + "entrypoint_command_handler_count": count_handlers_in_file(skill_dir / "scripts" / "yao.py"), + "command_module_count": len(command_module_paths(skill_dir)), "warn_line_threshold": warn_lines, "block_line_threshold": block_lines, "largest_file_lines": records[0]["lines"] if records else 0, @@ -167,6 +177,8 @@ def render_markdown(report: dict[str, Any]) -> str: f"- internal modules: `{summary['internal_module_count']}`", f"- CLI scripts: `{summary['cli_script_count']}`", f"- Yao CLI command handlers: `{summary['command_handler_count']}`", + f"- entrypoint command handlers: `{summary['entrypoint_command_handler_count']}`", + f"- command modules: `{summary['command_module_count']}`", f"- largest file lines: `{summary['largest_file_lines']}`", f"- hotspots: `{summary['hotspot_count']}`", f"- blockers: `{summary['blocker_count']}`", diff --git a/scripts/review_studio_data.py b/scripts/review_studio_data.py index 581b602c..f0dfd60f 100644 --- a/scripts/review_studio_data.py +++ b/scripts/review_studio_data.py @@ -246,7 +246,8 @@ def insight_cards(data: dict[str, dict[str, Any]]) -> list[dict[str, str]]: "value": str(architecture.get("hotspot_count", 0)), "detail": ( f"{architecture.get('largest_file_lines', 0)} largest lines; " - f"{architecture.get('command_handler_count', 0)} CLI handlers" + f"{architecture.get('command_handler_count', 0)} CLI handlers; " + f"{architecture.get('entrypoint_command_handler_count', 0)} in entrypoint" ), }, { diff --git a/scripts/review_studio_gates.py b/scripts/review_studio_gates.py index c8cbf57f..b9ed42a7 100644 --- a/scripts/review_studio_gates.py +++ b/scripts/review_studio_gates.py @@ -323,7 +323,8 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An f"{hotspot_count} {hotspot_label}; " f"{blocker_count} {blocker_label}; " f"largest {architecture_summary.get('largest_file_lines', 0)} lines; " - f"{architecture_summary.get('command_handler_count', 0)} CLI handlers" + f"{architecture_summary.get('command_handler_count', 0)} CLI handlers; " + f"{architecture_summary.get('entrypoint_command_handler_count', 0)} in entrypoint" ) gates.append( gate( diff --git a/scripts/yao.py b/scripts/yao.py index 4ee59352..9676e58d 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -7,12 +7,30 @@ from pathlib import Path from yao_cli_config import ( baseline_compare_args, local_output_runner_command, - provider_output_runner_command, resolve_promotion_target, resolve_target, ) from yao_cli_adaptation_commands import command_adapt_propose, command_adapt_scan from yao_cli_create_commands import command_init, command_quickstart +from yao_cli_distribution_commands import ( + command_compile_skill, + command_conformance, + command_install_simulate, + command_package, + command_package_verify, + command_registry_audit, + command_runtime_permissions, + command_skill_atlas, + command_skill_ir, + command_trust, + command_upgrade_check, +) +from yao_cli_output_commands import ( + command_output_eval, + command_output_execution, + command_output_review, + command_output_review_kit, +) from yao_cli_parser import build_parser as build_cli_parser from yao_cli_report_commands import ( command_artifact_design_profile, @@ -26,6 +44,7 @@ from yao_cli_report_commands import ( command_prompt_quality_profile, command_reference_scan, command_reference_synthesis, + command_report, command_review_studio, command_review_viewer, command_skill_os2_audit, @@ -146,99 +165,6 @@ def command_architecture_audit(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 -def command_report(args: argparse.Namespace) -> int: - steps = [] - if args.refresh_optimization: - steps.append(run_script("run_description_optimization_suite.py", [])) - steps.extend( - [ - run_script("build_confusion_matrix.py", []), - run_script("promotion_checker.py", []), - run_script("render_eval_dashboard.py", []), - run_script("render_intent_confidence.py", [str(ROOT)]), - run_script("render_description_drift_history.py", []), - run_script("render_iteration_ledger.py", []), - run_script("render_baseline_compare.py", baseline_compare_args()), - run_script("render_regression_history.py", []), - run_script("render_context_reports.py", []), - run_script("render_portability_report.py", []), - run_script("python_compat_check.py", [str(ROOT)]), - run_script("render_architecture_maintainability.py", [str(ROOT)]), - run_script("render_reference_synthesis.py", [str(ROOT)]), - run_script("render_artifact_design_profile.py", [str(ROOT)]), - run_script("render_prompt_quality_profile.py", [str(ROOT)]), - run_script("render_system_model.py", [str(ROOT)]), - run_script("compile_skill.py", [str(ROOT)]), - run_script("run_output_eval.py", []), - run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()]), - run_script("prepare_output_review_kit.py", []), - run_script("adjudicate_output_review.py", []), - run_script("render_adoption_drift_report.py", [str(ROOT)]), - run_script("render_telemetry_hook_recipes.py", [str(ROOT)]), - run_script("render_review_waivers.py", [str(ROOT)]), - run_script("render_review_annotations.py", [str(ROOT)]), - run_script("render_world_class_evidence_plan.py", [str(ROOT)]), - run_script("render_world_class_evidence_ledger.py", [str(ROOT)]), - run_script("render_world_class_evidence_intake.py", [str(ROOT)]), - run_script("render_world_class_submission_review.py", [str(ROOT)]), - run_script("render_world_class_operator_runbook.py", [str(ROOT)]), - run_script("render_world_class_claim_guard.py", [str(ROOT)]), - run_script("render_skill_os2_coverage.py", [str(ROOT)]), - run_script("render_benchmark_reproducibility.py", [str(ROOT)]), - run_script("render_skill_overview.py", [str(ROOT)]), - run_script("render_skill_interpretation.py", [str(ROOT)]), - run_script("render_evidence_consistency.py", [str(ROOT)]), - run_script("render_review_viewer.py", [str(ROOT)]), - ] - ) - report = { - "ok": all(step["ok"] for step in steps), - "steps": [{"command": step["command"], "ok": step["ok"], "returncode": step["returncode"]} for step in steps], - "artifacts": { - "eval_results": "reports/eval_suite.json", - "route_scorecard": "reports/route_scorecard.json", - "promotion_decisions": "reports/promotion_decisions.json", - "intent_confidence": "reports/intent-confidence.json", - "iteration_ledger": "reports/iteration_ledger.md", - "baseline_compare": "reports/baseline-compare.json", - "regression_history": "reports/regression_history.md", - "context_budget": "reports/context_budget.json", - "portability_score": "reports/portability_score.json", - "python_compatibility": "reports/python_compatibility.json", - "architecture_maintainability": "reports/architecture_maintainability.json", - "reference_synthesis": "reports/reference-synthesis.json", - "artifact_design_profile": "reports/artifact-design-profile.json", - "prompt_quality_profile": "reports/prompt-quality-profile.json", - "system_model": "reports/system-model.json", - "compiled_targets": "reports/compiled_targets.json", - "output_execution": "reports/output_execution_runs.json", - "output_review_kit": "reports/output_review_kit.json", - "output_review_kit_html": "reports/output_review_kit.html", - "output_review_adjudication": "reports/output_review_adjudication.json", - "adoption_drift": "reports/adoption_drift_report.json", - "telemetry_hooks": "reports/telemetry_hook_recipes.json", - "review_waivers": "reports/review_waivers.json", - "review_annotations": "reports/review_annotations.json", - "world_class_evidence_plan": "reports/world_class_evidence_plan.json", - "world_class_evidence_ledger": "reports/world_class_evidence_ledger.json", - "world_class_evidence_intake": "reports/world_class_evidence_intake.json", - "world_class_submission_review": "reports/world_class_submission_review.json", - "world_class_operator_runbook": "reports/world_class_operator_runbook.json", - "world_class_claim_guard": "reports/world_class_claim_guard.json", - "skill_os2_coverage": "reports/skill_os2_coverage.json", - "benchmark_reproducibility": "reports/benchmark_reproducibility.json", - "skill_overview": "reports/skill-overview.json", - "skill_interpretation": "reports/skill-interpretation.json", - "skill_interpretation_html": "reports/skill-interpretation.html", - "evidence_consistency": "reports/evidence_consistency.json", - "review_viewer": "reports/review-viewer.json", - "review_viewer_html": "reports/review-viewer.html", - }, - } - print(json.dumps(report, ensure_ascii=False, indent=2)) - return 0 if report["ok"] else 2 - - def command_feedback(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -438,257 +364,6 @@ def command_baseline_compare(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 -def command_skill_ir(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.validate_only: - cmd.append("--validate-only") - result = run_script("export_skill_ir.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_compile_skill(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - for target in args.target or []: - cmd.extend(["--target", target]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("compile_skill.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_eval(args: argparse.Namespace) -> int: - cmd = [] - if args.cases: - cmd.extend(["--cases", args.cases]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.blind_pack_json: - cmd.extend(["--blind-pack-json", args.blind_pack_json]) - if args.blind_pack_md: - cmd.extend(["--blind-pack-md", args.blind_pack_md]) - if args.blind_answer_key_json: - cmd.extend(["--blind-answer-key-json", args.blind_answer_key_json]) - result = run_script("run_output_eval.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_execution(args: argparse.Namespace) -> int: - cmd = [] - if args.cases: - cmd.extend(["--cases", args.cases]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.runner_command and args.provider_runner: - payload = { - "schema_version": "1.0", - "ok": False, - "failures": ["Use either --runner-command or --provider-runner, not both."], - } - print(json.dumps(payload, ensure_ascii=False, indent=2)) - return 2 - if args.provider_runner: - cmd.extend( - [ - "--runner-command", - provider_output_runner_command( - args.provider_runner, - model=args.provider_model, - base_url=args.provider_base_url, - api_key_env=args.api_key_env, - allow_insecure_localhost=args.allow_insecure_localhost, - allow_custom_base_url=args.allow_custom_base_url, - ), - ] - ) - elif args.runner_command: - cmd.extend(["--runner-command", args.runner_command]) - if args.timeout_seconds is not None: - cmd.extend(["--timeout-seconds", str(args.timeout_seconds)]) - result = run_script("run_output_execution.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_review_kit(args: argparse.Namespace) -> int: - cmd = [] - if args.blind_pack_json: - cmd.extend(["--blind-pack-json", args.blind_pack_json]) - if args.blind_pack_md: - cmd.extend(["--blind-pack-md", args.blind_pack_md]) - if args.decisions: - cmd.extend(["--decisions", args.decisions]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.output_html: - cmd.extend(["--output-html", args.output_html]) - if args.write_template: - cmd.append("--write-template") - result = run_script("prepare_output_review_kit.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_output_review(args: argparse.Namespace) -> int: - cmd = [] - if args.blind_pack: - cmd.extend(["--blind-pack", args.blind_pack]) - if args.answer_key: - cmd.extend(["--answer-key", args.answer_key]) - if args.decisions: - cmd.extend(["--decisions", args.decisions]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.write_template: - cmd.append("--write-template") - result = run_script("adjudicate_output_review.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_conformance(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - for target in args.target or []: - cmd.extend(["--target", target]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - result = run_script("run_conformance_suite.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_runtime_permissions(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.package_dir: - cmd.extend(["--package-dir", args.package_dir]) - for target in args.target or []: - cmd.extend(["--target", target]) - if getattr(args, "install_simulation_json", None): - cmd.extend(["--install-simulation-json", args.install_simulation_json]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - result = run_script("probe_runtime_permissions.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_trust(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - result = run_script("trust_check.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_skill_atlas(args: argparse.Namespace) -> int: - cmd = ["--workspace-root", str(Path(args.workspace_root).resolve())] - if args.output_dir: - cmd.extend(["--output-dir", args.output_dir]) - if args.report_html: - cmd.extend(["--report-html", args.report_html]) - if args.report_json: - cmd.extend(["--report-json", args.report_json]) - if args.overlap_threshold is not None: - cmd.extend(["--overlap-threshold", str(args.overlap_threshold)]) - if args.today: - cmd.extend(["--today", args.today]) - result = run_script("build_skill_atlas.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_registry_audit(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.registry_dir: - cmd.extend(["--registry-dir", args.registry_dir]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("registry_audit.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_package_verify(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.package_dir: - cmd.extend(["--package-dir", args.package_dir]) - if args.expectations: - cmd.extend(["--expectations", args.expectations]) - if args.registry_json: - cmd.extend(["--registry-json", args.registry_json]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.require_zip: - cmd.append("--require-zip") - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("verify_package.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_install_simulate(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - if args.package_dir: - cmd.extend(["--package-dir", args.package_dir]) - if args.install_root: - cmd.extend(["--install-root", args.install_root]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("simulate_install.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - -def command_upgrade_check(args: argparse.Namespace) -> int: - cmd = [str(Path(args.skill_dir).resolve())] - cmd.extend(["--previous-package-json", args.previous_package_json]) - if args.current_package_json: - cmd.extend(["--current-package-json", args.current_package_json]) - if args.output_json: - cmd.extend(["--output-json", args.output_json]) - if args.output_md: - cmd.extend(["--output-md", args.output_md]) - if args.generated_at: - cmd.extend(["--generated-at", args.generated_at]) - result = run_script("upgrade_check.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - def command_review(args: argparse.Namespace) -> int: target_name = resolve_promotion_target(args.target) bundle_dir = ROOT / "reports" / "iteration_bundles" / target_name @@ -814,23 +489,6 @@ def command_workspace_flow(args: argparse.Namespace) -> int: return 0 if report["ok"] else 2 -def command_package(args: argparse.Namespace) -> int: - cmd = [ - str(Path(args.skill_dir).resolve()), - "--output-dir", - args.output_dir, - ] - for platform in args.platform or ["generic"]: - cmd.extend(["--platform", platform]) - if args.expectations: - cmd.extend(["--expectations", args.expectations]) - if args.zip: - cmd.append("--zip") - result = run_script("cross_packager.py", cmd) - print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) - return 0 if result["ok"] else 2 - - def command_test(args: argparse.Namespace) -> int: proc = subprocess.run( ["make", args.target], diff --git a/scripts/yao_cli_distribution_commands.py b/scripts/yao_cli_distribution_commands.py new file mode 100644 index 00000000..95856e96 --- /dev/null +++ b/scripts/yao_cli_distribution_commands.py @@ -0,0 +1,164 @@ +"""Distribution, packaging, and runtime-gate command handlers for the Yao CLI.""" + +import argparse +import json +from pathlib import Path + +from yao_cli_runtime import run_script + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep distribution and runtime gate handlers outside the thin CLI orchestrator." + + +def emit_result(result: dict) -> int: + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_skill_ir(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.validate_only: + cmd.append("--validate-only") + return emit_result(run_script("export_skill_ir.py", cmd)) + + +def command_compile_skill(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + for target in args.target or []: + cmd.extend(["--target", target]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("compile_skill.py", cmd)) + + +def command_conformance(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + for target in args.target or []: + cmd.extend(["--target", target]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + return emit_result(run_script("run_conformance_suite.py", cmd)) + + +def command_runtime_permissions(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + for target in args.target or []: + cmd.extend(["--target", target]) + if getattr(args, "install_simulation_json", None): + cmd.extend(["--install-simulation-json", args.install_simulation_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + return emit_result(run_script("probe_runtime_permissions.py", cmd)) + + +def command_trust(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + return emit_result(run_script("trust_check.py", cmd)) + + +def command_skill_atlas(args: argparse.Namespace) -> int: + cmd = ["--workspace-root", str(Path(args.workspace_root).resolve())] + if args.output_dir: + cmd.extend(["--output-dir", args.output_dir]) + if args.report_html: + cmd.extend(["--report-html", args.report_html]) + if args.report_json: + cmd.extend(["--report-json", args.report_json]) + if args.overlap_threshold is not None: + cmd.extend(["--overlap-threshold", str(args.overlap_threshold)]) + if args.today: + cmd.extend(["--today", args.today]) + return emit_result(run_script("build_skill_atlas.py", cmd)) + + +def command_registry_audit(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.registry_dir: + cmd.extend(["--registry-dir", args.registry_dir]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("registry_audit.py", cmd)) + + +def command_package_verify(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + if args.expectations: + cmd.extend(["--expectations", args.expectations]) + if args.registry_json: + cmd.extend(["--registry-json", args.registry_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.require_zip: + cmd.append("--require-zip") + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("verify_package.py", cmd)) + + +def command_install_simulate(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + if args.package_dir: + cmd.extend(["--package-dir", args.package_dir]) + if args.install_root: + cmd.extend(["--install-root", args.install_root]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("simulate_install.py", cmd)) + + +def command_upgrade_check(args: argparse.Namespace) -> int: + cmd = [str(Path(args.skill_dir).resolve())] + cmd.extend(["--previous-package-json", args.previous_package_json]) + if args.current_package_json: + cmd.extend(["--current-package-json", args.current_package_json]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + return emit_result(run_script("upgrade_check.py", cmd)) + + +def command_package(args: argparse.Namespace) -> int: + cmd = [ + str(Path(args.skill_dir).resolve()), + "--output-dir", + args.output_dir, + ] + for platform in args.platform or ["generic"]: + cmd.extend(["--platform", platform]) + if args.expectations: + cmd.extend(["--expectations", args.expectations]) + if args.zip: + cmd.append("--zip") + return emit_result(run_script("cross_packager.py", cmd)) diff --git a/scripts/yao_cli_output_commands.py b/scripts/yao_cli_output_commands.py new file mode 100644 index 00000000..403119cd --- /dev/null +++ b/scripts/yao_cli_output_commands.py @@ -0,0 +1,106 @@ +"""Output evaluation and human-review command handlers for the Yao CLI.""" + +import argparse +import json + +from yao_cli_config import provider_output_runner_command +from yao_cli_runtime import run_script + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by yao.py to keep output evaluation and review handlers outside the thin CLI orchestrator." + + +def emit_result(result: dict) -> int: + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + +def command_output_eval(args: argparse.Namespace) -> int: + cmd = [] + if args.cases: + cmd.extend(["--cases", args.cases]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.blind_pack_json: + cmd.extend(["--blind-pack-json", args.blind_pack_json]) + if args.blind_pack_md: + cmd.extend(["--blind-pack-md", args.blind_pack_md]) + if args.blind_answer_key_json: + cmd.extend(["--blind-answer-key-json", args.blind_answer_key_json]) + return emit_result(run_script("run_output_eval.py", cmd)) + + +def command_output_execution(args: argparse.Namespace) -> int: + cmd = [] + if args.cases: + cmd.extend(["--cases", args.cases]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.runner_command and args.provider_runner: + payload = { + "schema_version": "1.0", + "ok": False, + "failures": ["Use either --runner-command or --provider-runner, not both."], + } + print(json.dumps(payload, ensure_ascii=False, indent=2)) + return 2 + if args.provider_runner: + cmd.extend( + [ + "--runner-command", + provider_output_runner_command( + args.provider_runner, + model=args.provider_model, + base_url=args.provider_base_url, + api_key_env=args.api_key_env, + allow_insecure_localhost=args.allow_insecure_localhost, + allow_custom_base_url=args.allow_custom_base_url, + ), + ] + ) + elif args.runner_command: + cmd.extend(["--runner-command", args.runner_command]) + if args.timeout_seconds is not None: + cmd.extend(["--timeout-seconds", str(args.timeout_seconds)]) + return emit_result(run_script("run_output_execution.py", cmd)) + + +def command_output_review_kit(args: argparse.Namespace) -> int: + cmd = [] + if args.blind_pack_json: + cmd.extend(["--blind-pack-json", args.blind_pack_json]) + if args.blind_pack_md: + cmd.extend(["--blind-pack-md", args.blind_pack_md]) + if args.decisions: + cmd.extend(["--decisions", args.decisions]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.output_html: + cmd.extend(["--output-html", args.output_html]) + if args.write_template: + cmd.append("--write-template") + return emit_result(run_script("prepare_output_review_kit.py", cmd)) + + +def command_output_review(args: argparse.Namespace) -> int: + cmd = [] + if args.blind_pack: + cmd.extend(["--blind-pack", args.blind_pack]) + if args.answer_key: + cmd.extend(["--answer-key", args.answer_key]) + if args.decisions: + cmd.extend(["--decisions", args.decisions]) + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.write_template: + cmd.append("--write-template") + return emit_result(run_script("adjudicate_output_review.py", cmd)) diff --git a/scripts/yao_cli_report_commands.py b/scripts/yao_cli_report_commands.py index 11d92e1f..8706c8ea 100644 --- a/scripts/yao_cli_report_commands.py +++ b/scripts/yao_cli_report_commands.py @@ -4,7 +4,8 @@ import argparse import json from pathlib import Path -from yao_cli_runtime import run_script +from yao_cli_config import baseline_compare_args, local_output_runner_command +from yao_cli_runtime import ROOT, run_script SCRIPT_INTERFACE = "internal-module" @@ -37,6 +38,99 @@ def render_skill_report_command(args: argparse.Namespace, script_name: str, *, m return emit_result(run_script(script_name, cmd)) +def command_report(args: argparse.Namespace) -> int: + steps = [] + if args.refresh_optimization: + steps.append(run_script("run_description_optimization_suite.py", [])) + steps.extend( + [ + run_script("build_confusion_matrix.py", []), + run_script("promotion_checker.py", []), + run_script("render_eval_dashboard.py", []), + run_script("render_intent_confidence.py", [str(ROOT)]), + run_script("render_description_drift_history.py", []), + run_script("render_iteration_ledger.py", []), + run_script("render_baseline_compare.py", baseline_compare_args()), + run_script("render_regression_history.py", []), + run_script("render_context_reports.py", []), + run_script("render_portability_report.py", []), + run_script("python_compat_check.py", [str(ROOT)]), + run_script("render_architecture_maintainability.py", [str(ROOT)]), + run_script("render_reference_synthesis.py", [str(ROOT)]), + run_script("render_artifact_design_profile.py", [str(ROOT)]), + run_script("render_prompt_quality_profile.py", [str(ROOT)]), + run_script("render_system_model.py", [str(ROOT)]), + run_script("compile_skill.py", [str(ROOT)]), + run_script("run_output_eval.py", []), + run_script("run_output_execution.py", ["--runner-command", local_output_runner_command()]), + run_script("prepare_output_review_kit.py", []), + run_script("adjudicate_output_review.py", []), + run_script("render_adoption_drift_report.py", [str(ROOT)]), + run_script("render_telemetry_hook_recipes.py", [str(ROOT)]), + run_script("render_review_waivers.py", [str(ROOT)]), + run_script("render_review_annotations.py", [str(ROOT)]), + run_script("render_world_class_evidence_plan.py", [str(ROOT)]), + run_script("render_world_class_evidence_ledger.py", [str(ROOT)]), + run_script("render_world_class_evidence_intake.py", [str(ROOT)]), + run_script("render_world_class_submission_review.py", [str(ROOT)]), + run_script("render_world_class_operator_runbook.py", [str(ROOT)]), + run_script("render_world_class_claim_guard.py", [str(ROOT)]), + run_script("render_skill_os2_coverage.py", [str(ROOT)]), + run_script("render_benchmark_reproducibility.py", [str(ROOT)]), + run_script("render_skill_overview.py", [str(ROOT)]), + run_script("render_skill_interpretation.py", [str(ROOT)]), + run_script("render_evidence_consistency.py", [str(ROOT)]), + run_script("render_review_viewer.py", [str(ROOT)]), + ] + ) + report = { + "ok": all(step["ok"] for step in steps), + "steps": [{"command": step["command"], "ok": step["ok"], "returncode": step["returncode"]} for step in steps], + "artifacts": { + "eval_results": "reports/eval_suite.json", + "route_scorecard": "reports/route_scorecard.json", + "promotion_decisions": "reports/promotion_decisions.json", + "intent_confidence": "reports/intent-confidence.json", + "iteration_ledger": "reports/iteration_ledger.md", + "baseline_compare": "reports/baseline-compare.json", + "regression_history": "reports/regression_history.md", + "context_budget": "reports/context_budget.json", + "portability_score": "reports/portability_score.json", + "python_compatibility": "reports/python_compatibility.json", + "architecture_maintainability": "reports/architecture_maintainability.json", + "reference_synthesis": "reports/reference-synthesis.json", + "artifact_design_profile": "reports/artifact-design-profile.json", + "prompt_quality_profile": "reports/prompt-quality-profile.json", + "system_model": "reports/system-model.json", + "compiled_targets": "reports/compiled_targets.json", + "output_execution": "reports/output_execution_runs.json", + "output_review_kit": "reports/output_review_kit.json", + "output_review_kit_html": "reports/output_review_kit.html", + "output_review_adjudication": "reports/output_review_adjudication.json", + "adoption_drift": "reports/adoption_drift_report.json", + "telemetry_hooks": "reports/telemetry_hook_recipes.json", + "review_waivers": "reports/review_waivers.json", + "review_annotations": "reports/review_annotations.json", + "world_class_evidence_plan": "reports/world_class_evidence_plan.json", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.json", + "world_class_evidence_intake": "reports/world_class_evidence_intake.json", + "world_class_submission_review": "reports/world_class_submission_review.json", + "world_class_operator_runbook": "reports/world_class_operator_runbook.json", + "world_class_claim_guard": "reports/world_class_claim_guard.json", + "skill_os2_coverage": "reports/skill_os2_coverage.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json", + "skill_overview": "reports/skill-overview.json", + "skill_interpretation": "reports/skill-interpretation.json", + "skill_interpretation_html": "reports/skill-interpretation.html", + "evidence_consistency": "reports/evidence_consistency.json", + "review_viewer": "reports/review-viewer.json", + "review_viewer_html": "reports/review-viewer.html", + }, + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + return 0 if report["ok"] else 2 + + def command_skill_report(args: argparse.Namespace) -> int: return render_skill_report_command(args, "render_skill_overview.py", markdown=False) diff --git a/tests/verify_architecture_maintainability.py b/tests/verify_architecture_maintainability.py index 137cb729..60ecb842 100644 --- a/tests/verify_architecture_maintainability.py +++ b/tests/verify_architecture_maintainability.py @@ -40,7 +40,9 @@ def main() -> None: assert payload["summary"]["decision"] == "pass", payload["summary"] assert payload["summary"]["hotspot_count"] == 0, payload["summary"] assert payload["summary"]["blocker_count"] == 0, payload["summary"] - assert 30 <= payload["summary"]["command_handler_count"] < 50, payload["summary"] + assert payload["summary"]["command_handler_count"] >= 60, payload["summary"] + assert payload["summary"]["entrypoint_command_handler_count"] < 30, payload["summary"] + assert payload["summary"]["command_module_count"] >= 5, payload["summary"] assert payload["summary"]["largest_file_lines"] < 900, payload["summary"] assert all(item["severity"] == "pass" for item in payload["largest_files"]), payload["largest_files"] renderer_lines = len((ROOT / "scripts" / "render_review_studio.py").read_text(encoding="utf-8").splitlines()) diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py index 8ad59bd0..01e22d1d 100644 --- a/tests/verify_trust_check.py +++ b/tests/verify_trust_check.py @@ -99,8 +99,12 @@ def main() -> None: "scripts/skill_report_world_class.py", "scripts/world_class_evidence_contract.py", "scripts/world_class_source_checks.py", + "scripts/yao_cli_adaptation_commands.py", "scripts/yao_cli_config.py", + "scripts/yao_cli_distribution_commands.py", + "scripts/yao_cli_output_commands.py", "scripts/yao_cli_parser.py", + "scripts/yao_cli_report_commands.py", "scripts/yao_cli_telemetry.py", ]: assert script_map[internal_module]["interface"] == "internal-module", script_map[internal_module] @@ -117,8 +121,12 @@ def main() -> None: assert "skill_report_world_class.py" not in warning_text, payload["warnings"] assert "world_class_evidence_contract.py" not in warning_text, payload["warnings"] assert "world_class_source_checks.py" not in warning_text, payload["warnings"] + assert "yao_cli_adaptation_commands.py" not in warning_text, payload["warnings"] assert "yao_cli_config.py" not in warning_text, payload["warnings"] + assert "yao_cli_distribution_commands.py" not in warning_text, payload["warnings"] + assert "yao_cli_output_commands.py" not in warning_text, payload["warnings"] assert "yao_cli_parser.py" not in warning_text, payload["warnings"] + assert "yao_cli_report_commands.py" not in warning_text, payload["warnings"] assert "yao_cli_telemetry.py" not in warning_text, payload["warnings"] assert "render_context_reports.py" not in warning_text, payload["warnings"] assert "render_social_preview.py" not in warning_text, payload["warnings"] diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 95c50d21..bd603984 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -13,6 +13,8 @@ sys.path.insert(0, str(ROOT / "scripts")) import yao as yao_cli_module # noqa: E402 import yao_cli_adaptation_commands # noqa: E402 import yao_cli_config # noqa: E402 +import yao_cli_distribution_commands # noqa: E402 +import yao_cli_output_commands # noqa: E402 import yao_cli_parser # noqa: E402 import yao_cli_report_commands # noqa: E402 import yao_cli_runtime # noqa: E402 @@ -72,10 +74,8 @@ def main() -> None: assert "--entry" in yao_cli_config.baseline_compare_args() assert "scripts/provider_output_eval_runner.py" in yao_cli_config.provider_output_runner_command("openai") assert "--allow-custom-base-url" in yao_cli_config.provider_output_runner_command("openai", allow_custom_base_url=True) - assert yao_cli_parser.SCRIPT_INTERFACE == "internal-module" - assert yao_cli_runtime.SCRIPT_INTERFACE == "internal-module" - assert yao_cli_adaptation_commands.SCRIPT_INTERFACE == "internal-module" - assert yao_cli_report_commands.SCRIPT_INTERFACE == "internal-module" + for module in (yao_cli_parser, yao_cli_runtime, yao_cli_adaptation_commands, yao_cli_distribution_commands, yao_cli_output_commands, yao_cli_report_commands): + assert module.SCRIPT_INTERFACE == "internal-module" assert callable(yao_cli_module.command_review_studio) parser_help = yao_cli_module.build_parser().format_help() expected_help = ( @@ -205,7 +205,8 @@ def main() -> None: assert architecture_result["ok"], architecture_result assert architecture_result["payload"]["summary"]["hotspot_count"] == 0, architecture_result assert architecture_result["payload"]["summary"]["blocker_count"] == 0, architecture_result - assert 30 <= architecture_result["payload"]["summary"]["command_handler_count"] < 55, architecture_result + assert architecture_result["payload"]["summary"]["command_handler_count"] >= 60, architecture_result + assert architecture_result["payload"]["summary"]["entrypoint_command_handler_count"] < 30, architecture_result world_class_evidence_result = run( "world-class-evidence",