Extract Yao CLI test helpers

This commit is contained in:
yaojingang
2026-06-16 21:42:08 +08:00
parent 1ae595bdd5
commit c86602332d
10 changed files with 170 additions and 136 deletions
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-16T13:40:09Z",
"skill_dir": ".",
"source_patterns": "reports/user_patterns.json",
"pattern_count": 5,
+1 -1
View File
@@ -1,6 +1,6 @@
# Adaptation Proposals
- Generated at: `2026-06-16`
- Generated at: `2026-06-16T13:40:09Z`
- Pattern report: `reports/user_patterns.json`
- Proposal only: `true`
- Writes repository files: `false`
+10 -17
View File
@@ -4,9 +4,9 @@
"generated_at": "2026-06-16",
"skill_dir": ".",
"summary": {
"python_file_count": 202,
"python_file_count": 203,
"script_file_count": 131,
"test_file_count": 71,
"test_file_count": 72,
"internal_module_count": 48,
"cli_script_count": 85,
"command_handler_count": 68,
@@ -16,7 +16,7 @@
"watch_line_threshold": 720,
"block_line_threshold": 1500,
"largest_file_lines": 852,
"watchlist_count": 2,
"watchlist_count": 1,
"hotspot_count": 0,
"blocker_count": 0,
"decision": "pass"
@@ -29,13 +29,6 @@
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "tests/verify_yao_cli.py",
"lines": 787,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/trust_check.py",
"lines": 714,
@@ -57,6 +50,13 @@
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "tests/verify_yao_cli.py",
"lines": 696,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/world_class_evidence_contract.py",
"lines": 686,
@@ -114,13 +114,6 @@
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "tests/verify_yao_cli.py",
"lines": 787,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
}
],
"hotspots": [],
+4 -5
View File
@@ -5,9 +5,9 @@ Generated at: `2026-06-16`
## Summary
- decision: `pass`
- python files: `202`
- python files: `203`
- scripts: `131`
- tests: `71`
- tests: `72`
- internal modules: `48`
- CLI scripts: `85`
- Yao CLI command handlers: `68`
@@ -15,7 +15,7 @@ Generated at: `2026-06-16`
- command modules: `6`
- largest file lines: `852`
- watch threshold lines: `720`
- watchlist: `2`
- watchlist: `1`
- hotspots: `0`
- blockers: `0`
@@ -30,17 +30,16 @@ No file-size hotspots found.
| File | Lines | Kind | Recommended next split |
| --- | ---: | --- | --- |
| `tests/verify_review_studio.py` | `852` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
| `tests/verify_yao_cli.py` | `787` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
## Largest Files
| File | Lines | Kind | Severity |
| --- | ---: | --- | --- |
| `tests/verify_review_studio.py` | `852` | `test` | `pass` |
| `tests/verify_yao_cli.py` | `787` | `test` | `pass` |
| `scripts/trust_check.py` | `714` | `internal-module` | `pass` |
| `scripts/review_studio_gates.py` | `707` | `internal-module` | `pass` |
| `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` |
| `tests/verify_yao_cli.py` | `696` | `test` | `pass` |
| `scripts/world_class_evidence_contract.py` | `686` | `internal-module` | `pass` |
| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` |
| `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` |
+3 -3
View File
@@ -6,9 +6,9 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 797,
"other_text_tokens": 1052655,
"other_text_tokens": 1052729,
"estimated_initial_load_tokens": 990,
"estimated_total_text_tokens": 1053452,
"estimated_total_text_tokens": 1053526,
"deferred_resource_tokens": 487270,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
@@ -66,7 +66,7 @@
],
"summary": "Large deferred resources are indexed and backed by evidence."
},
"relevant_file_count": 650,
"relevant_file_count": 652,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 131.3
+7 -1
View File
@@ -5,7 +5,7 @@
"root": ".",
"summary": {
"target_python": "3.11",
"file_count": 205,
"file_count": 206,
"issue_count": 0,
"syntax_error_count": 0,
"fstring_311_violation_count": 0,
@@ -1246,6 +1246,12 @@
"issue_count": 0,
"issues": []
},
{
"path": "tests/yao_cli_helpers.py",
"ok": true,
"issue_count": 0,
"issues": []
},
{
"path": "tests/yao_cli_report_refresh.py",
"ok": true,
+1 -1
View File
@@ -6,7 +6,7 @@ Generated at: `2026-06-16`
- decision: `pass`
- target python: `3.11`
- files scanned: `205`
- files scanned: `206`
- issues: `0`
- syntax errors: `0`
- f-string 3.11 violations: `0`
+3 -1
View File
@@ -48,7 +48,9 @@ def main() -> None:
assert payload["summary"]["largest_file_lines"] < 900, payload["summary"]
assert all(item["severity"] == "pass" for item in payload["largest_files"]), payload["largest_files"]
assert all(item["severity"] == "pass" for item in payload["watchlist"]), payload["watchlist"]
assert any(item["path"] == "tests/verify_yao_cli.py" for item in payload["watchlist"]), payload["watchlist"]
watchlist_paths = {item["path"] for item in payload["watchlist"]}
assert "tests/verify_yao_cli.py" not in watchlist_paths, payload["watchlist"]
assert "tests/verify_review_studio.py" in watchlist_paths, payload["watchlist"]
renderer_lines = len((ROOT / "scripts" / "render_review_studio.py").read_text(encoding="utf-8").splitlines())
action_module = (ROOT / "scripts" / "review_studio_actions.py").read_text(encoding="utf-8")
action_lines = len(action_module.splitlines())
+15 -106
View File
@@ -1,64 +1,20 @@
#!/usr/bin/env python3
import json
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CLI = ROOT / "scripts" / "yao.py"
BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan"
sys.path.insert(0, str(ROOT / "scripts"))
import yao as yao_cli_module # noqa: E402
import yao_cli_adaptation_commands # noqa: E402
import yao_cli_config # noqa: E402
import yao_cli_distribution_commands # noqa: E402
import yao_cli_output_commands # noqa: E402
import yao_cli_parser # noqa: E402
import yao_cli_parser_evidence # noqa: E402
import yao_cli_parser_operations # noqa: E402
import yao_cli_report_commands # noqa: E402
import yao_cli_runtime # noqa: E402
from yao_cli_report_refresh import refresh_root_report_consistency_inputs # noqa: E402
def run(*args: str, input_text: str | None = None) -> dict:
env = dict(os.environ)
env["YAO_CLI_TELEMETRY"] = "0"
env.pop("YAO_CLI_TELEMETRY_EVENTS", None)
proc = subprocess.run(
[sys.executable, str(CLI), *args],
cwd=ROOT,
capture_output=True,
text=True,
input=input_text,
env=env,
)
payload = json.loads(proc.stdout)
return {
"ok": proc.returncode == 0,
"returncode": proc.returncode,
"payload": payload,
"stderr": proc.stderr,
}
def run_with_env(extra_env: dict[str, str], *args: str) -> dict:
env = dict(os.environ)
env.update(extra_env)
proc = subprocess.run(
[sys.executable, str(CLI), *args],
cwd=ROOT,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(proc.stdout)
return {
"ok": proc.returncode == 0,
"returncode": proc.returncode,
"payload": payload,
"stderr": proc.stderr,
}
from yao_cli_helpers import (
BENCHMARK_FIXTURE_DIR,
ROOT,
assert_cli_module_contracts,
assert_created_skill_reports,
assert_creation_report_view,
assert_help_surface,
refresh_root_report_consistency_inputs,
run,
run_with_env,
)
def main() -> None:
@@ -68,64 +24,17 @@ def main() -> None:
tmp_root.mkdir(parents=True, exist_ok=True)
remote_version = tmp_root / "remote-version.txt"
remote_version.write_text("9.9.9\n", encoding="utf-8")
assert yao_cli_config.resolve_target("root")["title"] == "Root Description Optimization"
assert yao_cli_config.resolve_promotion_target("root") == "yao-meta-skill"
assert yao_cli_config.infer_archetype("Standardize team review workflow.", "")[0] == "production"
assert yao_cli_config.infer_archetype("Govern release policy.", "")[0] == "governed"
assert "--entry" in yao_cli_config.baseline_compare_args()
assert "scripts/provider_output_eval_runner.py" in yao_cli_config.provider_output_runner_command("openai")
assert "--allow-custom-base-url" in yao_cli_config.provider_output_runner_command("openai", allow_custom_base_url=True)
for module in (
yao_cli_parser,
yao_cli_parser_evidence,
yao_cli_parser_operations,
yao_cli_runtime,
yao_cli_adaptation_commands,
yao_cli_distribution_commands,
yao_cli_output_commands,
yao_cli_report_commands,
):
assert module.SCRIPT_INTERFACE == "internal-module"
assert callable(yao_cli_module.command_review_studio)
parser_help = yao_cli_module.build_parser().format_help()
expected_help = (
"quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage "
"world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit world-class-submission-review world-class-runbook world-class-claim-guard "
"benchmark-reproducibility evidence-consistency output-review-kit output-review-import adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry"
).split()
assert all(item in parser_help for item in expected_help), parser_help
assert_cli_module_contracts()
assert_help_surface()
init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root))
assert init_result["ok"], init_result
created = Path(init_result["payload"]["root"])
assert (created / "SKILL.md").exists(), created
assert (created / "README.md").exists(), created
expected_reports = [
"intent-dialogue.md", "intent-confidence.md", "skill-overview.html", "skill-interpretation.html",
"skill-interpretation.json", "review-studio.html", "review-studio.json", "review-viewer.html",
"reference-scan.md", "reference-synthesis.md", "output-risk-profile.md", "artifact-design-profile.md",
"prompt-quality-profile.md", "system-model.md", "skill-ir.json", "compiled_targets.md",
"compiled_targets.json", "iteration-directions.md", "adoption_drift_report.md",
"adoption_drift_report.json", "review_waivers.md", "review_waivers.json", "review_annotations.md",
"review_annotations.json",
]
assert all((created / "reports" / path).exists() for path in expected_reports), created
assert_created_skill_reports(created)
assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created
init_report_view = init_result["payload"]["report_view"]
assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view
assert init_report_view["interpretation_report"].endswith("reports/skill-interpretation.html"), init_report_view
assert Path(init_report_view["html_report"]).exists(), init_report_view
assert Path(init_report_view["interpretation_report"]).exists(), init_report_view
assert init_report_view["review_studio"].endswith("reports/review-studio.html"), init_report_view
assert Path(init_report_view["review_studio"]).exists(), init_report_view
assert "Skill 已创建完成" in init_report_view["message"], init_report_view
assert "reports/skill-interpretation.html" in init_report_view["message"], init_report_view
assert "Review Studio 2.0" in init_report_view["message"], init_report_view
assert "目标编译" in init_report_view["message"], init_report_view
assert "reports/compiled_targets.md" in init_report_view["message"], init_report_view
assert "概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view
assert "默认使用中文简体" in init_report_view["message"], init_report_view
assert "切换英文版" in init_report_view["message"], init_report_view
assert_creation_report_view(init_result["payload"]["report_view"])
init_skill_ir = init_result["payload"]["skill_ir"]
assert init_skill_ir["name"] == "cli-demo-skill", init_skill_ir
assert init_skill_ir["trigger_samples"] >= 1, init_skill_ir
+125
View File
@@ -0,0 +1,125 @@
import json
import os
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CLI = ROOT / "scripts" / "yao.py"
BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan"
sys.path.insert(0, str(ROOT / "scripts"))
import yao as yao_cli_module # noqa: E402
import yao_cli_adaptation_commands # noqa: E402
import yao_cli_config # noqa: E402
import yao_cli_distribution_commands # noqa: E402
import yao_cli_output_commands # noqa: E402
import yao_cli_parser # noqa: E402
import yao_cli_parser_evidence # noqa: E402
import yao_cli_parser_operations # noqa: E402
import yao_cli_report_commands # noqa: E402
import yao_cli_runtime # noqa: E402
from yao_cli_report_refresh import refresh_root_report_consistency_inputs # noqa: E402
def run(*args: str, input_text: str | None = None) -> dict:
env = dict(os.environ)
env["YAO_CLI_TELEMETRY"] = "0"
env.pop("YAO_CLI_TELEMETRY_EVENTS", None)
proc = subprocess.run(
[sys.executable, str(CLI), *args],
cwd=ROOT,
capture_output=True,
text=True,
input=input_text,
env=env,
)
payload = json.loads(proc.stdout)
return {
"ok": proc.returncode == 0,
"returncode": proc.returncode,
"payload": payload,
"stderr": proc.stderr,
}
def run_with_env(extra_env: dict[str, str], *args: str) -> dict:
env = dict(os.environ)
env.update(extra_env)
proc = subprocess.run(
[sys.executable, str(CLI), *args],
cwd=ROOT,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(proc.stdout)
return {
"ok": proc.returncode == 0,
"returncode": proc.returncode,
"payload": payload,
"stderr": proc.stderr,
}
def assert_cli_module_contracts() -> None:
assert yao_cli_config.resolve_target("root")["title"] == "Root Description Optimization"
assert yao_cli_config.resolve_promotion_target("root") == "yao-meta-skill"
assert yao_cli_config.infer_archetype("Standardize team review workflow.", "")[0] == "production"
assert yao_cli_config.infer_archetype("Govern release policy.", "")[0] == "governed"
assert "--entry" in yao_cli_config.baseline_compare_args()
assert "scripts/provider_output_eval_runner.py" in yao_cli_config.provider_output_runner_command("openai")
assert "--allow-custom-base-url" in yao_cli_config.provider_output_runner_command("openai", allow_custom_base_url=True)
for module in (
yao_cli_parser,
yao_cli_parser_evidence,
yao_cli_parser_operations,
yao_cli_runtime,
yao_cli_adaptation_commands,
yao_cli_distribution_commands,
yao_cli_output_commands,
yao_cli_report_commands,
):
assert module.SCRIPT_INTERFACE == "internal-module"
assert callable(yao_cli_module.command_review_studio)
def assert_help_surface() -> None:
parser_help = yao_cli_module.build_parser().format_help()
expected_help = (
"quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage "
"world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit world-class-submission-review world-class-runbook world-class-claim-guard "
"benchmark-reproducibility evidence-consistency output-review-kit output-review-import adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry"
).split()
assert all(item in parser_help for item in expected_help), parser_help
def assert_created_skill_reports(created: Path) -> None:
expected_reports = [
"intent-dialogue.md", "intent-confidence.md", "skill-overview.html", "skill-interpretation.html",
"skill-interpretation.json", "review-studio.html", "review-studio.json", "review-viewer.html",
"reference-scan.md", "reference-synthesis.md", "output-risk-profile.md", "artifact-design-profile.md",
"prompt-quality-profile.md", "system-model.md", "skill-ir.json", "compiled_targets.md",
"compiled_targets.json", "iteration-directions.md", "adoption_drift_report.md",
"adoption_drift_report.json", "review_waivers.md", "review_waivers.json", "review_annotations.md",
"review_annotations.json",
]
assert all((created / "reports" / path).exists() for path in expected_reports), created
def assert_creation_report_view(report_view: dict) -> None:
assert report_view["html_report"].endswith("reports/skill-overview.html"), report_view
assert report_view["interpretation_report"].endswith("reports/skill-interpretation.html"), report_view
assert Path(report_view["html_report"]).exists(), report_view
assert Path(report_view["interpretation_report"]).exists(), report_view
assert report_view["review_studio"].endswith("reports/review-studio.html"), report_view
assert Path(report_view["review_studio"]).exists(), report_view
assert "Skill 已创建完成" in report_view["message"], report_view
assert "reports/skill-interpretation.html" in report_view["message"], report_view
assert "Review Studio 2.0" in report_view["message"], report_view
assert "目标编译" in report_view["message"], report_view
assert "reports/compiled_targets.md" in report_view["message"], report_view
assert "概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线" in report_view["message"], report_view
assert "默认使用中文简体" in report_view["message"], report_view
assert "切换英文版" in report_view["message"], report_view