From de51f71ec378d820f0f25d247223dbcd82160b73 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 21:54:29 +0800 Subject: [PATCH] Extract Review Studio test helpers --- reports/architecture_maintainability.json | 28 +- reports/architecture_maintainability.md | 14 +- reports/context_budget.json | 6 +- reports/python_compatibility.json | 8 +- reports/python_compatibility.md | 2 +- tests/review_studio_helpers.py | 166 +++++++++++ tests/verify_architecture_maintainability.py | 10 +- tests/verify_review_studio.py | 284 +------------------ 8 files changed, 201 insertions(+), 317 deletions(-) create mode 100644 tests/review_studio_helpers.py diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index f4a88463..4edf7a92 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,9 +4,9 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 203, + "python_file_count": 204, "script_file_count": 131, - "test_file_count": 72, + "test_file_count": 73, "internal_module_count": 48, "cli_script_count": 85, "command_handler_count": 68, @@ -15,20 +15,13 @@ "warn_line_threshold": 900, "watch_line_threshold": 720, "block_line_threshold": 1500, - "largest_file_lines": 852, - "watchlist_count": 1, + "largest_file_lines": 714, + "watchlist_count": 0, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" }, "largest_files": [ - { - "path": "tests/verify_review_studio.py", - "lines": 852, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/trust_check.py", "lines": 714, @@ -105,17 +98,16 @@ "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." - } - ], - "watchlist": [ + }, { - "path": "tests/verify_review_studio.py", - "lines": 852, - "kind": "test", + "path": "scripts/cross_packager.py", + "lines": 638, + "kind": "cli-script", "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." } ], + "watchlist": [], "hotspots": [], "actions": [], "artifacts": { diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 887711b8..97ca201f 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,17 +5,17 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `203` +- python files: `204` - scripts: `131` -- tests: `72` +- tests: `73` - internal modules: `48` - CLI scripts: `85` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` - command modules: `6` -- largest file lines: `852` +- largest file lines: `714` - watch threshold lines: `720` -- watchlist: `1` +- watchlist: `0` - hotspots: `0` - blockers: `0` @@ -27,15 +27,12 @@ No file-size hotspots found. ## Watchlist -| File | Lines | Kind | Recommended next split | -| --- | ---: | --- | --- | -| `tests/verify_review_studio.py` | `852` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | +No near-threshold files found. ## Largest Files | File | Lines | Kind | Severity | | --- | ---: | --- | --- | -| `tests/verify_review_studio.py` | `852` | `test` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | | `scripts/review_studio_gates.py` | `707` | `internal-module` | `pass` | | `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` | @@ -47,6 +44,7 @@ No file-size hotspots found. | `scripts/render_skill_os2_coverage.py` | `649` | `cli-script` | `pass` | | `scripts/render_review_studio.py` | `647` | `cli-script` | `pass` | | `scripts/render_reference_synthesis.py` | `644` | `cli-script` | `pass` | +| `scripts/cross_packager.py` | `638` | `cli-script` | `pass` | ## Release Rule diff --git a/reports/context_budget.json b/reports/context_budget.json index 9d778a78..62614502 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,9 +6,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1052732, + "other_text_tokens": 1051797, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1053529, + "estimated_total_text_tokens": 1052594, "deferred_resource_tokens": 487270, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ @@ -66,7 +66,7 @@ ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 652, + "relevant_file_count": 654, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 2313586d..47b7fb5d 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 206, + "file_count": 207, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -826,6 +826,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/review_studio_helpers.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_adaptation_safety.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 25de45f6..4b0f16ed 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `206` +- files scanned: `207` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/tests/review_studio_helpers.py b/tests/review_studio_helpers.py new file mode 100644 index 00000000..cf276299 --- /dev/null +++ b/tests/review_studio_helpers.py @@ -0,0 +1,166 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPTS = ROOT / "scripts" +SCRIPT = SCRIPTS / "render_review_studio.py" + + +def run(args: list[object]) -> subprocess.CompletedProcess[str]: + return subprocess.run([str(item) for item in args], cwd=ROOT, check=True, capture_output=True, text=True) + + +def run_script(script_name: str, *args: object) -> subprocess.CompletedProcess[str]: + return run([sys.executable, SCRIPTS / script_name, *args]) + + +def prepare_tmp_root() -> Path: + tmp_root = ROOT / "tests" / "tmp_review_studio" + if tmp_root.exists(): + shutil.rmtree(tmp_root) + tmp_root.mkdir(parents=True, exist_ok=True) + return tmp_root + + +def copy_tmp_report(tmp_root: Path, stem: str) -> None: + for suffix in ("json", "md"): + (ROOT / "reports" / f"{stem}.{suffix}").write_text( + (tmp_root / f"{stem}.{suffix}").read_text(encoding="utf-8"), + encoding="utf-8", + ) + + +def prepare_review_studio_inputs(tmp_root: Path) -> None: + run_script("run_output_eval.py") + run_script("prepare_output_review_kit.py") + run_script("run_output_execution.py", "--runner-command", json.dumps(["python3", "scripts/local_output_eval_runner.py"])) + run_script("adjudicate_output_review.py") + run_script("compile_skill.py", ROOT, "--generated-at", "2026-06-13") + + package_dir = tmp_root / "dist" + run_script( + "cross_packager.py", + ROOT, + "--platform", + "openai", + "--platform", + "claude", + "--platform", + "generic", + "--platform", + "vscode", + "--expectations", + ROOT / "evals" / "packaging_expectations.json", + "--output-dir", + package_dir, + "--zip", + ) + run_script( + "simulate_install.py", + ROOT, + "--package-dir", + package_dir, + "--install-root", + tmp_root / "install-root", + "--output-json", + tmp_root / "install_simulation.json", + "--output-md", + tmp_root / "install_simulation.md", + "--generated-at", + "2026-06-13", + ) + copy_tmp_report(tmp_root, "install_simulation") + + run_script("registry_audit.py", ROOT, "--generated-at", "2026-06-13") + run_script("render_intent_confidence.py", ROOT) + run_script("trust_check.py", ROOT) + run_script("python_compat_check.py", ROOT, "--generated-at", "2026-06-13") + run_script("render_architecture_maintainability.py", ROOT, "--generated-at", "2026-06-13") + run_script("render_context_reports.py") + run_script( + "render_adoption_drift_report.py", + ROOT, + "--events-jsonl", + tmp_root / "telemetry_events.jsonl", + "--record-event", + "skill_activation", + "--activation-type", + "explicit", + "--outcome", + "accepted", + "--timestamp", + "2026-06-13T10:00:00Z", + ) + run_script( + "build_skill_atlas.py", + "--workspace-root", + ROOT, + "--output-dir", + ROOT / "skill_atlas", + "--report-html", + ROOT / "reports" / "skill_atlas.html", + "--report-json", + ROOT / "reports" / "skill_atlas.json", + "--today", + "2026-06-13", + ) + run_script( + "upgrade_check.py", + ROOT, + "--previous-package-json", + ROOT / "registry" / "examples" / "yao-meta-skill-1.0.0.json", + "--current-package-json", + ROOT / "reports" / "registry_audit.json", + "--output-json", + tmp_root / "upgrade_check.json", + "--output-md", + tmp_root / "upgrade_check.md", + "--generated-at", + "2026-06-13", + ) + copy_tmp_report(tmp_root, "upgrade_check") + + run_script("render_review_waivers.py", ROOT, "--generated-at", "2026-06-13") + run_script( + "render_review_annotations.py", + ROOT, + "--annotations-json", + tmp_root / "empty_review_annotations_input.json", + ) + run_script("probe_runtime_permissions.py", ROOT, "--package-dir", package_dir) + + for script_name in [ + "render_skill_os2_audit.py", + "render_world_class_evidence_plan.py", + "render_world_class_evidence_ledger.py", + "render_world_class_evidence_intake.py", + "render_world_class_preflight.py", + "render_world_class_submission_review.py", + "render_world_class_operator_runbook.py", + "render_world_class_claim_guard.py", + "render_skill_os2_coverage.py", + ]: + run_script(script_name, ROOT, "--generated-at", "2026-06-13") + + +def render_review_studio_fixture(tmp_root: Path) -> tuple[Path, Path, subprocess.CompletedProcess[str]]: + prepare_review_studio_inputs(tmp_root) + output_html = tmp_root / "review-studio.html" + output_json = tmp_root / "review-studio.json" + proc = run( + [ + sys.executable, + SCRIPT, + ROOT, + "--output-html", + output_html, + "--output-json", + output_json, + ] + ) + return output_html, output_json, proc diff --git a/tests/verify_architecture_maintainability.py b/tests/verify_architecture_maintainability.py index 294f4eee..8f0553fd 100644 --- a/tests/verify_architecture_maintainability.py +++ b/tests/verify_architecture_maintainability.py @@ -39,7 +39,7 @@ def main() -> None: assert payload["ok"], payload assert payload["summary"]["decision"] == "pass", payload["summary"] assert payload["summary"]["watch_line_threshold"] == 720, payload["summary"] - assert payload["summary"]["watchlist_count"] > 0, payload["summary"] + assert payload["summary"]["watchlist_count"] == 0, payload["summary"] assert payload["summary"]["hotspot_count"] == 0, payload["summary"] assert payload["summary"]["blocker_count"] == 0, payload["summary"] assert payload["summary"]["command_handler_count"] >= 60, payload["summary"] @@ -47,10 +47,7 @@ def main() -> None: assert payload["summary"]["command_module_count"] >= 5, payload["summary"] assert payload["summary"]["largest_file_lines"] < 900, payload["summary"] assert all(item["severity"] == "pass" for item in payload["largest_files"]), payload["largest_files"] - assert all(item["severity"] == "pass" for item in payload["watchlist"]), payload["watchlist"] - watchlist_paths = {item["path"] for item in payload["watchlist"]} - assert "tests/verify_yao_cli.py" not in watchlist_paths, payload["watchlist"] - assert "tests/verify_review_studio.py" in watchlist_paths, payload["watchlist"] + assert payload["watchlist"] == [], payload["watchlist"] renderer_lines = len((ROOT / "scripts" / "render_review_studio.py").read_text(encoding="utf-8").splitlines()) action_module = (ROOT / "scripts" / "review_studio_actions.py").read_text(encoding="utf-8") action_lines = len(action_module.splitlines()) @@ -66,8 +63,7 @@ def main() -> None: markdown = output_md.read_text(encoding="utf-8") assert "# Architecture Maintainability" in markdown, markdown assert "No file-size hotspots found." in markdown, markdown - assert "## Watchlist" in markdown, markdown - assert "tests/verify_yao_cli.py" in markdown, markdown + assert "No near-threshold files found." in markdown, markdown assert "Do not split a file only for line count" in markdown, markdown blocker_proc = subprocess.run( diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 904abd88..02d528c7 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -1,14 +1,11 @@ #!/usr/bin/env python3 import json import re -import shutil -import subprocess import sys -from pathlib import Path -ROOT = Path(__file__).resolve().parent.parent -SCRIPT = ROOT / "scripts" / "render_review_studio.py" +from review_studio_helpers import ROOT, prepare_tmp_root, render_review_studio_fixture + sys.path.insert(0, str(ROOT / "scripts")) import render_review_studio as review_studio # noqa: E402 import review_studio_formatting as review_formatting # noqa: E402 @@ -17,279 +14,8 @@ import review_studio_layout as review_layout # noqa: E402 def main() -> None: - tmp_root = ROOT / "tests" / "tmp_review_studio" - if tmp_root.exists(): - shutil.rmtree(tmp_root) - tmp_root.mkdir(parents=True, exist_ok=True) - subprocess.run([sys.executable, str(ROOT / "scripts" / "run_output_eval.py")], cwd=ROOT, check=True, capture_output=True, text=True) - subprocess.run([sys.executable, str(ROOT / "scripts" / "prepare_output_review_kit.py")], cwd=ROOT, check=True, capture_output=True, text=True) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "run_output_execution.py"), - "--runner-command", - json.dumps(["python3", "scripts/local_output_eval_runner.py"]), - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "adjudicate_output_review.py")], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "compile_skill.py"), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - package_dir = tmp_root / "dist" - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "cross_packager.py"), - str(ROOT), - "--platform", - "openai", - "--platform", - "claude", - "--platform", - "generic", - "--platform", - "vscode", - "--expectations", - str(ROOT / "evals" / "packaging_expectations.json"), - "--output-dir", - str(package_dir), - "--zip", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "simulate_install.py"), - str(ROOT), - "--package-dir", - str(package_dir), - "--install-root", - str(tmp_root / "install-root"), - "--output-json", - str(tmp_root / "install_simulation.json"), - "--output-md", - str(tmp_root / "install_simulation.md"), - "--generated-at", - "2026-06-13", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - (ROOT / "reports" / "install_simulation.json").write_text( - (tmp_root / "install_simulation.json").read_text(encoding="utf-8"), - encoding="utf-8", - ) - (ROOT / "reports" / "install_simulation.md").write_text( - (tmp_root / "install_simulation.md").read_text(encoding="utf-8"), - encoding="utf-8", - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "registry_audit.py"), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "render_intent_confidence.py"), str(ROOT)], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "trust_check.py"), str(ROOT)], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "python_compat_check.py"), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "render_architecture_maintainability.py"), - str(ROOT), - "--generated-at", - "2026-06-13", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "render_context_reports.py")], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "render_adoption_drift_report.py"), - str(ROOT), - "--events-jsonl", - str(tmp_root / "telemetry_events.jsonl"), - "--record-event", - "skill_activation", - "--activation-type", - "explicit", - "--outcome", - "accepted", - "--timestamp", - "2026-06-13T10:00:00Z", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "build_skill_atlas.py"), - "--workspace-root", - str(ROOT), - "--output-dir", - str(ROOT / "skill_atlas"), - "--report-html", - str(ROOT / "reports" / "skill_atlas.html"), - "--report-json", - str(ROOT / "reports" / "skill_atlas.json"), - "--today", - "2026-06-13", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "upgrade_check.py"), - str(ROOT), - "--previous-package-json", - str(ROOT / "registry" / "examples" / "yao-meta-skill-1.0.0.json"), - "--current-package-json", - str(ROOT / "reports" / "registry_audit.json"), - "--output-json", - str(tmp_root / "upgrade_check.json"), - "--output-md", - str(tmp_root / "upgrade_check.md"), - "--generated-at", - "2026-06-13", - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - (ROOT / "reports" / "upgrade_check.json").write_text( - (tmp_root / "upgrade_check.json").read_text(encoding="utf-8"), - encoding="utf-8", - ) - (ROOT / "reports" / "upgrade_check.md").write_text( - (tmp_root / "upgrade_check.md").read_text(encoding="utf-8"), - encoding="utf-8", - ) - subprocess.run( - [sys.executable, str(ROOT / "scripts" / "render_review_waivers.py"), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "render_review_annotations.py"), - str(ROOT), - "--annotations-json", - str(tmp_root / "empty_review_annotations_input.json"), - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - subprocess.run( - [ - sys.executable, - str(ROOT / "scripts" / "probe_runtime_permissions.py"), - str(ROOT), - "--package-dir", - str(package_dir), - ], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - for script_name in [ - "render_skill_os2_audit.py", - "render_world_class_evidence_plan.py", - "render_world_class_evidence_ledger.py", - "render_world_class_evidence_intake.py", - "render_world_class_preflight.py", - "render_world_class_submission_review.py", - "render_world_class_operator_runbook.py", - "render_world_class_claim_guard.py", - "render_skill_os2_coverage.py", - ]: - subprocess.run( - [sys.executable, str(ROOT / "scripts" / script_name), str(ROOT), "--generated-at", "2026-06-13"], - cwd=ROOT, - check=True, - capture_output=True, - text=True, - ) - - output_html = tmp_root / "review-studio.html" - output_json = tmp_root / "review-studio.json" - proc = subprocess.run( - [ - sys.executable, - str(SCRIPT), - str(ROOT), - "--output-html", - str(output_html), - "--output-json", - str(output_json), - ], - cwd=ROOT, - capture_output=True, - text=True, - check=True, - ) + tmp_root = prepare_tmp_root() + output_html, output_json, proc = render_review_studio_fixture(tmp_root) payload = json.loads(proc.stdout) assert payload["ok"], payload assert payload["schema_version"] == "2.0", payload @@ -508,7 +234,7 @@ def main() -> None: assert full_payload["data"]["python_compatibility"]["summary"]["target_python"] == "3.11", full_payload["data"]["python_compatibility"] assert full_payload["data"]["python_compatibility"]["summary"]["issue_count"] == 0, full_payload["data"]["python_compatibility"] assert full_payload["data"]["architecture_maintainability"]["summary"]["hotspot_count"] == 0, full_payload["data"]["architecture_maintainability"] - assert full_payload["data"]["architecture_maintainability"]["summary"]["watchlist_count"] > 0, full_payload["data"]["architecture_maintainability"] + assert full_payload["data"]["architecture_maintainability"]["summary"]["watchlist_count"] == 0, full_payload["data"]["architecture_maintainability"] assert full_payload["data"]["architecture_maintainability"]["summary"]["blocker_count"] == 0, full_payload["data"]["architecture_maintainability"] action_keys = {item["gate_key"] for item in full_payload["review_actions"]} assert action_keys == {"output-lab", "review-waivers", "world-class-evidence"}, full_payload["review_actions"]