Extract Yao CLI test helpers
This commit is contained in:
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-16",
|
||||
"generated_at": "2026-06-16T13:40:09Z",
|
||||
"skill_dir": ".",
|
||||
"source_patterns": "reports/user_patterns.json",
|
||||
"pattern_count": 5,
|
||||
|
||||
@@ -1,6 +1,6 @@
|
||||
# Adaptation Proposals
|
||||
|
||||
- Generated at: `2026-06-16`
|
||||
- Generated at: `2026-06-16T13:40:09Z`
|
||||
- Pattern report: `reports/user_patterns.json`
|
||||
- Proposal only: `true`
|
||||
- Writes repository files: `false`
|
||||
|
||||
@@ -4,9 +4,9 @@
|
||||
"generated_at": "2026-06-16",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"python_file_count": 202,
|
||||
"python_file_count": 203,
|
||||
"script_file_count": 131,
|
||||
"test_file_count": 71,
|
||||
"test_file_count": 72,
|
||||
"internal_module_count": 48,
|
||||
"cli_script_count": 85,
|
||||
"command_handler_count": 68,
|
||||
@@ -16,7 +16,7 @@
|
||||
"watch_line_threshold": 720,
|
||||
"block_line_threshold": 1500,
|
||||
"largest_file_lines": 852,
|
||||
"watchlist_count": 2,
|
||||
"watchlist_count": 1,
|
||||
"hotspot_count": 0,
|
||||
"blocker_count": 0,
|
||||
"decision": "pass"
|
||||
@@ -29,13 +29,6 @@
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_yao_cli.py",
|
||||
"lines": 787,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "scripts/trust_check.py",
|
||||
"lines": 714,
|
||||
@@ -57,6 +50,13 @@
|
||||
"severity": "pass",
|
||||
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_yao_cli.py",
|
||||
"lines": 696,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "scripts/world_class_evidence_contract.py",
|
||||
"lines": 686,
|
||||
@@ -114,13 +114,6 @@
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
},
|
||||
{
|
||||
"path": "tests/verify_yao_cli.py",
|
||||
"lines": 787,
|
||||
"kind": "test",
|
||||
"severity": "pass",
|
||||
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
|
||||
}
|
||||
],
|
||||
"hotspots": [],
|
||||
|
||||
@@ -5,9 +5,9 @@ Generated at: `2026-06-16`
|
||||
## Summary
|
||||
|
||||
- decision: `pass`
|
||||
- python files: `202`
|
||||
- python files: `203`
|
||||
- scripts: `131`
|
||||
- tests: `71`
|
||||
- tests: `72`
|
||||
- internal modules: `48`
|
||||
- CLI scripts: `85`
|
||||
- Yao CLI command handlers: `68`
|
||||
@@ -15,7 +15,7 @@ Generated at: `2026-06-16`
|
||||
- command modules: `6`
|
||||
- largest file lines: `852`
|
||||
- watch threshold lines: `720`
|
||||
- watchlist: `2`
|
||||
- watchlist: `1`
|
||||
- hotspots: `0`
|
||||
- blockers: `0`
|
||||
|
||||
@@ -30,17 +30,16 @@ No file-size hotspots found.
|
||||
| File | Lines | Kind | Recommended next split |
|
||||
| --- | ---: | --- | --- |
|
||||
| `tests/verify_review_studio.py` | `852` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
| `tests/verify_yao_cli.py` | `787` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
|
||||
|
||||
## Largest Files
|
||||
|
||||
| File | Lines | Kind | Severity |
|
||||
| --- | ---: | --- | --- |
|
||||
| `tests/verify_review_studio.py` | `852` | `test` | `pass` |
|
||||
| `tests/verify_yao_cli.py` | `787` | `test` | `pass` |
|
||||
| `scripts/trust_check.py` | `714` | `internal-module` | `pass` |
|
||||
| `scripts/review_studio_gates.py` | `707` | `internal-module` | `pass` |
|
||||
| `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` |
|
||||
| `tests/verify_yao_cli.py` | `696` | `test` | `pass` |
|
||||
| `scripts/world_class_evidence_contract.py` | `686` | `internal-module` | `pass` |
|
||||
| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` |
|
||||
| `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` |
|
||||
|
||||
@@ -6,9 +6,9 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 797,
|
||||
"other_text_tokens": 1052655,
|
||||
"other_text_tokens": 1052729,
|
||||
"estimated_initial_load_tokens": 990,
|
||||
"estimated_total_text_tokens": 1053452,
|
||||
"estimated_total_text_tokens": 1053526,
|
||||
"deferred_resource_tokens": 487270,
|
||||
"deferred_resource_warn_threshold": 120000,
|
||||
"deferred_resource_dirs": [
|
||||
@@ -66,7 +66,7 @@
|
||||
],
|
||||
"summary": "Large deferred resources are indexed and backed by evidence."
|
||||
},
|
||||
"relevant_file_count": 650,
|
||||
"relevant_file_count": 652,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
"quality_density": 131.3
|
||||
|
||||
@@ -5,7 +5,7 @@
|
||||
"root": ".",
|
||||
"summary": {
|
||||
"target_python": "3.11",
|
||||
"file_count": 205,
|
||||
"file_count": 206,
|
||||
"issue_count": 0,
|
||||
"syntax_error_count": 0,
|
||||
"fstring_311_violation_count": 0,
|
||||
@@ -1246,6 +1246,12 @@
|
||||
"issue_count": 0,
|
||||
"issues": []
|
||||
},
|
||||
{
|
||||
"path": "tests/yao_cli_helpers.py",
|
||||
"ok": true,
|
||||
"issue_count": 0,
|
||||
"issues": []
|
||||
},
|
||||
{
|
||||
"path": "tests/yao_cli_report_refresh.py",
|
||||
"ok": true,
|
||||
|
||||
@@ -6,7 +6,7 @@ Generated at: `2026-06-16`
|
||||
|
||||
- decision: `pass`
|
||||
- target python: `3.11`
|
||||
- files scanned: `205`
|
||||
- files scanned: `206`
|
||||
- issues: `0`
|
||||
- syntax errors: `0`
|
||||
- f-string 3.11 violations: `0`
|
||||
|
||||
@@ -48,7 +48,9 @@ def main() -> None:
|
||||
assert payload["summary"]["largest_file_lines"] < 900, payload["summary"]
|
||||
assert all(item["severity"] == "pass" for item in payload["largest_files"]), payload["largest_files"]
|
||||
assert all(item["severity"] == "pass" for item in payload["watchlist"]), payload["watchlist"]
|
||||
assert any(item["path"] == "tests/verify_yao_cli.py" for item in payload["watchlist"]), payload["watchlist"]
|
||||
watchlist_paths = {item["path"] for item in payload["watchlist"]}
|
||||
assert "tests/verify_yao_cli.py" not in watchlist_paths, payload["watchlist"]
|
||||
assert "tests/verify_review_studio.py" in watchlist_paths, payload["watchlist"]
|
||||
renderer_lines = len((ROOT / "scripts" / "render_review_studio.py").read_text(encoding="utf-8").splitlines())
|
||||
action_module = (ROOT / "scripts" / "review_studio_actions.py").read_text(encoding="utf-8")
|
||||
action_lines = len(action_module.splitlines())
|
||||
|
||||
+15
-106
@@ -1,64 +1,20 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
CLI = ROOT / "scripts" / "yao.py"
|
||||
BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan"
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
import yao as yao_cli_module # noqa: E402
|
||||
import yao_cli_adaptation_commands # noqa: E402
|
||||
import yao_cli_config # noqa: E402
|
||||
import yao_cli_distribution_commands # noqa: E402
|
||||
import yao_cli_output_commands # noqa: E402
|
||||
import yao_cli_parser # noqa: E402
|
||||
import yao_cli_parser_evidence # noqa: E402
|
||||
import yao_cli_parser_operations # noqa: E402
|
||||
import yao_cli_report_commands # noqa: E402
|
||||
import yao_cli_runtime # noqa: E402
|
||||
from yao_cli_report_refresh import refresh_root_report_consistency_inputs # noqa: E402
|
||||
|
||||
def run(*args: str, input_text: str | None = None) -> dict:
|
||||
env = dict(os.environ)
|
||||
env["YAO_CLI_TELEMETRY"] = "0"
|
||||
env.pop("YAO_CLI_TELEMETRY_EVENTS", None)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(CLI), *args],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
input=input_text,
|
||||
env=env,
|
||||
)
|
||||
payload = json.loads(proc.stdout)
|
||||
return {
|
||||
"ok": proc.returncode == 0,
|
||||
"returncode": proc.returncode,
|
||||
"payload": payload,
|
||||
"stderr": proc.stderr,
|
||||
}
|
||||
|
||||
|
||||
def run_with_env(extra_env: dict[str, str], *args: str) -> dict:
|
||||
env = dict(os.environ)
|
||||
env.update(extra_env)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(CLI), *args],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
payload = json.loads(proc.stdout)
|
||||
return {
|
||||
"ok": proc.returncode == 0,
|
||||
"returncode": proc.returncode,
|
||||
"payload": payload,
|
||||
"stderr": proc.stderr,
|
||||
}
|
||||
from yao_cli_helpers import (
|
||||
BENCHMARK_FIXTURE_DIR,
|
||||
ROOT,
|
||||
assert_cli_module_contracts,
|
||||
assert_created_skill_reports,
|
||||
assert_creation_report_view,
|
||||
assert_help_surface,
|
||||
refresh_root_report_consistency_inputs,
|
||||
run,
|
||||
run_with_env,
|
||||
)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
@@ -68,64 +24,17 @@ def main() -> None:
|
||||
tmp_root.mkdir(parents=True, exist_ok=True)
|
||||
remote_version = tmp_root / "remote-version.txt"
|
||||
remote_version.write_text("9.9.9\n", encoding="utf-8")
|
||||
assert yao_cli_config.resolve_target("root")["title"] == "Root Description Optimization"
|
||||
assert yao_cli_config.resolve_promotion_target("root") == "yao-meta-skill"
|
||||
assert yao_cli_config.infer_archetype("Standardize team review workflow.", "")[0] == "production"
|
||||
assert yao_cli_config.infer_archetype("Govern release policy.", "")[0] == "governed"
|
||||
assert "--entry" in yao_cli_config.baseline_compare_args()
|
||||
assert "scripts/provider_output_eval_runner.py" in yao_cli_config.provider_output_runner_command("openai")
|
||||
assert "--allow-custom-base-url" in yao_cli_config.provider_output_runner_command("openai", allow_custom_base_url=True)
|
||||
for module in (
|
||||
yao_cli_parser,
|
||||
yao_cli_parser_evidence,
|
||||
yao_cli_parser_operations,
|
||||
yao_cli_runtime,
|
||||
yao_cli_adaptation_commands,
|
||||
yao_cli_distribution_commands,
|
||||
yao_cli_output_commands,
|
||||
yao_cli_report_commands,
|
||||
):
|
||||
assert module.SCRIPT_INTERFACE == "internal-module"
|
||||
assert callable(yao_cli_module.command_review_studio)
|
||||
parser_help = yao_cli_module.build_parser().format_help()
|
||||
expected_help = (
|
||||
"quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage "
|
||||
"world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit world-class-submission-review world-class-runbook world-class-claim-guard "
|
||||
"benchmark-reproducibility evidence-consistency output-review-kit output-review-import adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry"
|
||||
).split()
|
||||
assert all(item in parser_help for item in expected_help), parser_help
|
||||
assert_cli_module_contracts()
|
||||
assert_help_surface()
|
||||
|
||||
init_result = run("init", "cli-demo-skill", "--description", "CLI demo skill.", "--output-dir", str(tmp_root))
|
||||
assert init_result["ok"], init_result
|
||||
created = Path(init_result["payload"]["root"])
|
||||
assert (created / "SKILL.md").exists(), created
|
||||
assert (created / "README.md").exists(), created
|
||||
expected_reports = [
|
||||
"intent-dialogue.md", "intent-confidence.md", "skill-overview.html", "skill-interpretation.html",
|
||||
"skill-interpretation.json", "review-studio.html", "review-studio.json", "review-viewer.html",
|
||||
"reference-scan.md", "reference-synthesis.md", "output-risk-profile.md", "artifact-design-profile.md",
|
||||
"prompt-quality-profile.md", "system-model.md", "skill-ir.json", "compiled_targets.md",
|
||||
"compiled_targets.json", "iteration-directions.md", "adoption_drift_report.md",
|
||||
"adoption_drift_report.json", "review_waivers.md", "review_waivers.json", "review_annotations.md",
|
||||
"review_annotations.json",
|
||||
]
|
||||
assert all((created / "reports" / path).exists() for path in expected_reports), created
|
||||
assert_created_skill_reports(created)
|
||||
assert "Honest Boundaries" in (created / "SKILL.md").read_text(encoding="utf-8"), created
|
||||
init_report_view = init_result["payload"]["report_view"]
|
||||
assert init_report_view["html_report"].endswith("reports/skill-overview.html"), init_report_view
|
||||
assert init_report_view["interpretation_report"].endswith("reports/skill-interpretation.html"), init_report_view
|
||||
assert Path(init_report_view["html_report"]).exists(), init_report_view
|
||||
assert Path(init_report_view["interpretation_report"]).exists(), init_report_view
|
||||
assert init_report_view["review_studio"].endswith("reports/review-studio.html"), init_report_view
|
||||
assert Path(init_report_view["review_studio"]).exists(), init_report_view
|
||||
assert "Skill 已创建完成" in init_report_view["message"], init_report_view
|
||||
assert "reports/skill-interpretation.html" in init_report_view["message"], init_report_view
|
||||
assert "Review Studio 2.0" in init_report_view["message"], init_report_view
|
||||
assert "目标编译" in init_report_view["message"], init_report_view
|
||||
assert "reports/compiled_targets.md" in init_report_view["message"], init_report_view
|
||||
assert "概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线" in init_report_view["message"], init_report_view
|
||||
assert "默认使用中文简体" in init_report_view["message"], init_report_view
|
||||
assert "切换英文版" in init_report_view["message"], init_report_view
|
||||
assert_creation_report_view(init_result["payload"]["report_view"])
|
||||
init_skill_ir = init_result["payload"]["skill_ir"]
|
||||
assert init_skill_ir["name"] == "cli-demo-skill", init_skill_ir
|
||||
assert init_skill_ir["trigger_samples"] >= 1, init_skill_ir
|
||||
|
||||
@@ -0,0 +1,125 @@
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
CLI = ROOT / "scripts" / "yao.py"
|
||||
BENCHMARK_FIXTURE_DIR = ROOT / "tests" / "fixtures" / "github_benchmark_scan"
|
||||
|
||||
sys.path.insert(0, str(ROOT / "scripts"))
|
||||
import yao as yao_cli_module # noqa: E402
|
||||
import yao_cli_adaptation_commands # noqa: E402
|
||||
import yao_cli_config # noqa: E402
|
||||
import yao_cli_distribution_commands # noqa: E402
|
||||
import yao_cli_output_commands # noqa: E402
|
||||
import yao_cli_parser # noqa: E402
|
||||
import yao_cli_parser_evidence # noqa: E402
|
||||
import yao_cli_parser_operations # noqa: E402
|
||||
import yao_cli_report_commands # noqa: E402
|
||||
import yao_cli_runtime # noqa: E402
|
||||
from yao_cli_report_refresh import refresh_root_report_consistency_inputs # noqa: E402
|
||||
|
||||
|
||||
def run(*args: str, input_text: str | None = None) -> dict:
|
||||
env = dict(os.environ)
|
||||
env["YAO_CLI_TELEMETRY"] = "0"
|
||||
env.pop("YAO_CLI_TELEMETRY_EVENTS", None)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(CLI), *args],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
input=input_text,
|
||||
env=env,
|
||||
)
|
||||
payload = json.loads(proc.stdout)
|
||||
return {
|
||||
"ok": proc.returncode == 0,
|
||||
"returncode": proc.returncode,
|
||||
"payload": payload,
|
||||
"stderr": proc.stderr,
|
||||
}
|
||||
|
||||
|
||||
def run_with_env(extra_env: dict[str, str], *args: str) -> dict:
|
||||
env = dict(os.environ)
|
||||
env.update(extra_env)
|
||||
proc = subprocess.run(
|
||||
[sys.executable, str(CLI), *args],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
payload = json.loads(proc.stdout)
|
||||
return {
|
||||
"ok": proc.returncode == 0,
|
||||
"returncode": proc.returncode,
|
||||
"payload": payload,
|
||||
"stderr": proc.stderr,
|
||||
}
|
||||
|
||||
|
||||
def assert_cli_module_contracts() -> None:
|
||||
assert yao_cli_config.resolve_target("root")["title"] == "Root Description Optimization"
|
||||
assert yao_cli_config.resolve_promotion_target("root") == "yao-meta-skill"
|
||||
assert yao_cli_config.infer_archetype("Standardize team review workflow.", "")[0] == "production"
|
||||
assert yao_cli_config.infer_archetype("Govern release policy.", "")[0] == "governed"
|
||||
assert "--entry" in yao_cli_config.baseline_compare_args()
|
||||
assert "scripts/provider_output_eval_runner.py" in yao_cli_config.provider_output_runner_command("openai")
|
||||
assert "--allow-custom-base-url" in yao_cli_config.provider_output_runner_command("openai", allow_custom_base_url=True)
|
||||
for module in (
|
||||
yao_cli_parser,
|
||||
yao_cli_parser_evidence,
|
||||
yao_cli_parser_operations,
|
||||
yao_cli_runtime,
|
||||
yao_cli_adaptation_commands,
|
||||
yao_cli_distribution_commands,
|
||||
yao_cli_output_commands,
|
||||
yao_cli_report_commands,
|
||||
):
|
||||
assert module.SCRIPT_INTERFACE == "internal-module"
|
||||
assert callable(yao_cli_module.command_review_studio)
|
||||
|
||||
|
||||
def assert_help_surface() -> None:
|
||||
parser_help = yao_cli_module.build_parser().format_help()
|
||||
expected_help = (
|
||||
"quickstart skill-interpretation review-studio python-compat architecture-audit skill-os2-audit skill-os2-coverage "
|
||||
"world-class-evidence world-class-ledger world-class-intake world-class-preflight world-class-submission-kit world-class-submission-review world-class-runbook world-class-claim-guard "
|
||||
"benchmark-reproducibility evidence-consistency output-review-kit output-review-import adapt-scan adapt-propose adapt-apply telemetry-import telemetry-emit telemetry-hooks weekly-curator --record-cli-telemetry"
|
||||
).split()
|
||||
assert all(item in parser_help for item in expected_help), parser_help
|
||||
|
||||
|
||||
def assert_created_skill_reports(created: Path) -> None:
|
||||
expected_reports = [
|
||||
"intent-dialogue.md", "intent-confidence.md", "skill-overview.html", "skill-interpretation.html",
|
||||
"skill-interpretation.json", "review-studio.html", "review-studio.json", "review-viewer.html",
|
||||
"reference-scan.md", "reference-synthesis.md", "output-risk-profile.md", "artifact-design-profile.md",
|
||||
"prompt-quality-profile.md", "system-model.md", "skill-ir.json", "compiled_targets.md",
|
||||
"compiled_targets.json", "iteration-directions.md", "adoption_drift_report.md",
|
||||
"adoption_drift_report.json", "review_waivers.md", "review_waivers.json", "review_annotations.md",
|
||||
"review_annotations.json",
|
||||
]
|
||||
assert all((created / "reports" / path).exists() for path in expected_reports), created
|
||||
|
||||
|
||||
def assert_creation_report_view(report_view: dict) -> None:
|
||||
assert report_view["html_report"].endswith("reports/skill-overview.html"), report_view
|
||||
assert report_view["interpretation_report"].endswith("reports/skill-interpretation.html"), report_view
|
||||
assert Path(report_view["html_report"]).exists(), report_view
|
||||
assert Path(report_view["interpretation_report"]).exists(), report_view
|
||||
assert report_view["review_studio"].endswith("reports/review-studio.html"), report_view
|
||||
assert Path(report_view["review_studio"]).exists(), report_view
|
||||
assert "Skill 已创建完成" in report_view["message"], report_view
|
||||
assert "reports/skill-interpretation.html" in report_view["message"], report_view
|
||||
assert "Review Studio 2.0" in report_view["message"], report_view
|
||||
assert "目标编译" in report_view["message"], report_view
|
||||
assert "reports/compiled_targets.md" in report_view["message"], report_view
|
||||
assert "概述、指标、原理、触发边界、输入输出、目标编译、质量评估、风险治理、包体资产和升级路线" in report_view["message"], report_view
|
||||
assert "默认使用中文简体" in report_view["message"], report_view
|
||||
assert "切换英文版" in report_view["message"], report_view
|
||||
Reference in New Issue
Block a user