wehub-resource-sync
720 lines
35 KiB
Python
720 lines
35 KiB
Python
#!/usr/bin/env python3
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
sys.path.insert(0, str(ROOT / "scripts"))
|
|
from evidence_consistency_release import CLEAN_LOCK_REPORT_COMMANDS, SOURCE_REFRESH_REPORT_COMMANDS
|
|
|
|
SCRIPT = ROOT / "scripts" / "render_evidence_consistency.py"
|
|
TMP = ROOT / "tests" / "tmp_evidence_consistency"
|
|
REPORT_FILES = [
|
|
"AGENTS.md",
|
|
"reports/benchmark_reproducibility.json",
|
|
"reports/skill-overview.json",
|
|
"reports/skill-overview.html",
|
|
"reports/skill-interpretation.json",
|
|
"reports/skill-interpretation.html",
|
|
"reports/adoption_drift_report.json",
|
|
"reports/world_class_evidence_ledger.json",
|
|
"reports/world_class_evidence_plan.json",
|
|
"reports/world_class_evidence_intake.json",
|
|
"reports/world_class_evidence_preflight.json",
|
|
"reports/world_class_evidence_preflight.html",
|
|
"reports/world_class_submission_review.json",
|
|
"reports/world_class_operator_runbook.json",
|
|
"reports/skill_os2_coverage.json",
|
|
"reports/review-studio.json",
|
|
"reports/package_verification.json",
|
|
"reports/install_simulation.json",
|
|
"reports/security_trust_report.json",
|
|
"reports/context_budget.json",
|
|
"reports/world_class_claim_guard.json",
|
|
"reports/skill-os-2-review.md",
|
|
"skill-ir/examples/yao-meta-skill.json",
|
|
"scripts/ci_test.py",
|
|
]
|
|
|
|
|
|
def run(cmd: list[str], *, check: bool = False) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True, check=check)
|
|
|
|
|
|
def copy_reports(dst: Path) -> None:
|
|
for relative in REPORT_FILES:
|
|
source = ROOT / relative
|
|
target = dst / relative
|
|
target.parent.mkdir(parents=True, exist_ok=True)
|
|
shutil.copy2(source, target)
|
|
|
|
|
|
def refresh_embedded_reports() -> None:
|
|
script_names = [
|
|
"render_context_reports.py",
|
|
"render_skill_os2_coverage.py",
|
|
"render_world_class_evidence_plan.py",
|
|
"render_world_class_evidence_ledger.py",
|
|
"render_world_class_evidence_intake.py",
|
|
"render_world_class_preflight.py",
|
|
"render_world_class_submission_review.py",
|
|
"render_world_class_operator_runbook.py",
|
|
"render_world_class_claim_guard.py",
|
|
"render_benchmark_reproducibility.py",
|
|
"render_skill_overview.py",
|
|
"render_skill_interpretation.py",
|
|
"render_review_viewer.py",
|
|
"render_review_studio.py",
|
|
]
|
|
allow_nonzero = {
|
|
"render_world_class_evidence_intake.py",
|
|
"render_world_class_submission_review.py",
|
|
}
|
|
for script_name in script_names:
|
|
command = [sys.executable, str(ROOT / "scripts" / script_name)]
|
|
if script_name != "render_context_reports.py":
|
|
command.append(str(ROOT))
|
|
proc = subprocess.run(
|
|
command,
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
)
|
|
if proc.returncode != 0 and script_name not in allow_nonzero:
|
|
raise subprocess.CalledProcessError(proc.returncode, command, proc.stdout, proc.stderr)
|
|
|
|
|
|
def assert_world_class_roadmap_matches_ledger() -> None:
|
|
ledger = json.loads((ROOT / "reports" / "world_class_evidence_ledger.json").read_text(encoding="utf-8"))
|
|
summary = ledger["summary"]
|
|
pending_count = int(summary["pending_count"])
|
|
external_pending_count = int(summary["external_pending_count"])
|
|
human_pending_count = int(summary["human_pending_count"])
|
|
expected_total = f"{pending_count} 项待补证据"
|
|
expected_breakdown = f"外部 {external_pending_count} 项、人工 {human_pending_count} 项"
|
|
|
|
for report_name in ["skill-overview", "skill-interpretation"]:
|
|
report_path = ROOT / "reports" / f"{report_name}.json"
|
|
report = json.loads(report_path.read_text(encoding="utf-8"))
|
|
serialized = json.dumps(report, ensure_ascii=False)
|
|
assert "继续补齐剩余 2 项外部/人工证据" not in serialized, report_name
|
|
assert report["world_class_readiness"]["pending_count"] == pending_count, report["world_class_readiness"]
|
|
assert report["world_class_readiness"]["external_pending_count"] == external_pending_count, report[
|
|
"world_class_readiness"
|
|
]
|
|
assert report["world_class_readiness"]["human_pending_count"] == human_pending_count, report[
|
|
"world_class_readiness"
|
|
]
|
|
actions = "\n".join(report["iteration_roadmap"]["items"][0]["actions"])
|
|
if pending_count > 2:
|
|
assert expected_total in actions, actions
|
|
assert expected_breakdown in actions, actions
|
|
|
|
|
|
def assert_release_evidence_instructions_cover_first_class_reports() -> None:
|
|
agents_text = (ROOT / "AGENTS.md").read_text(encoding="utf-8")
|
|
source_refresh_header = "After source changes that affect scripts"
|
|
clean_lock_header = "For final release evidence"
|
|
assert source_refresh_header in agents_text, agents_text
|
|
assert clean_lock_header in agents_text, agents_text
|
|
|
|
source_refresh = agents_text.split(source_refresh_header, 1)[1].split(clean_lock_header, 1)[0]
|
|
clean_lock = agents_text.split(clean_lock_header, 1)[1].split("If `reports/benchmark_reproducibility.json`", 1)[0]
|
|
for command in SOURCE_REFRESH_REPORT_COMMANDS:
|
|
assert command in source_refresh, command
|
|
for command in CLEAN_LOCK_REPORT_COMMANDS:
|
|
assert command in clean_lock, command
|
|
|
|
|
|
def main() -> None:
|
|
shutil.rmtree(TMP, ignore_errors=True)
|
|
TMP.mkdir(parents=True, exist_ok=True)
|
|
refresh_embedded_reports()
|
|
assert_world_class_roadmap_matches_ledger()
|
|
assert_release_evidence_instructions_cover_first_class_reports()
|
|
output_json = TMP / "evidence_consistency.json"
|
|
output_md = TMP / "evidence_consistency.md"
|
|
proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(ROOT),
|
|
"--output-json",
|
|
str(output_json),
|
|
"--output-md",
|
|
str(output_md),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
],
|
|
check=True,
|
|
)
|
|
payload = json.loads(proc.stdout)
|
|
assert payload["schema_version"] == "1.0", payload
|
|
assert payload["ok"] is True, payload
|
|
assert payload["summary"]["decision"] == "consistent", payload
|
|
assert payload["summary"]["fail_count"] == 0, payload
|
|
assert payload["summary"]["check_count"] >= 31, payload
|
|
checks = {item["key"]: item for item in payload["checks"]}
|
|
assert checks["release-evidence-flow-covers-first-class-reports"]["status"] == "pass", checks[
|
|
"release-evidence-flow-covers-first-class-reports"
|
|
]
|
|
assert checks["review-studio-context-budget-mirror"]["status"] == "pass", checks[
|
|
"review-studio-context-budget-mirror"
|
|
]
|
|
assert checks["skill-ir-evidence-path-contract"]["status"] == "pass", checks[
|
|
"skill-ir-evidence-path-contract"
|
|
]
|
|
assert checks["skill-ir-evidence-path-contract"]["actual"]["review_studio_evidence_path"] == (
|
|
"skill-ir/examples/yao-meta-skill.json"
|
|
), checks["skill-ir-evidence-path-contract"]
|
|
assert checks["review-studio-gate-action-mirror"]["status"] == "pass", checks[
|
|
"review-studio-gate-action-mirror"
|
|
]
|
|
gate_action_mirror = checks["review-studio-gate-action-mirror"]["actual"]
|
|
assert gate_action_mirror["non_pass_gate_keys"] == [
|
|
"output-lab",
|
|
"review-waivers",
|
|
"world-class-evidence",
|
|
], gate_action_mirror
|
|
assert gate_action_mirror["action_gate_keys"] == gate_action_mirror["non_pass_gate_keys"], gate_action_mirror
|
|
assert gate_action_mirror["pass_gate_action_ids_empty"] is True, gate_action_mirror
|
|
assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["source_ref_present"] is True, (
|
|
gate_action_mirror
|
|
)
|
|
assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["title_present"] is True, (
|
|
gate_action_mirror
|
|
)
|
|
assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["next_step_present"] is True, (
|
|
gate_action_mirror
|
|
)
|
|
assert gate_action_mirror["gate_action_mirrors"]["world-class-evidence"]["reason_present"] is True, (
|
|
gate_action_mirror
|
|
)
|
|
assert checks["overview-benchmark-summary"]["status"] == "pass", checks["overview-benchmark-summary"]
|
|
assert checks["interpretation-adoption-summary"]["status"] == "pass", checks["interpretation-adoption-summary"]
|
|
assert checks["coverage-world-class-boundary"]["status"] == "pass", checks["coverage-world-class-boundary"]
|
|
assert checks["benchmark-beta-public-claim-split"]["status"] == "pass", checks[
|
|
"benchmark-beta-public-claim-split"
|
|
]
|
|
beta_split = checks["benchmark-beta-public-claim-split"]["actual"]
|
|
assert beta_split["public_claim_ready"] is False, beta_split
|
|
assert beta_split["human_review_complete"] is False, beta_split
|
|
assert beta_split["deferred_evidence_keys"] == ["human-adjudication", "native-client-telemetry", "native-permission-enforcement", "provider-holdout"], beta_split
|
|
assert beta_split["deferred_human_review"] is True, beta_split
|
|
assert checks["preflight-world-class-boundary"]["status"] == "pass", checks["preflight-world-class-boundary"]
|
|
assert checks["preflight-submission-kit-handoff"]["status"] == "pass", checks[
|
|
"preflight-submission-kit-handoff"
|
|
]
|
|
preflight_handoff = checks["preflight-submission-kit-handoff"]["actual"]
|
|
assert preflight_handoff["artifact_role_source"] == "world-class-submission-kit", preflight_handoff
|
|
assert preflight_handoff["artifact_role_counts_as_evidence"] is False, preflight_handoff
|
|
assert preflight_handoff["submission_ref_role_present"] is True, preflight_handoff
|
|
assert preflight_handoff["supporting_evidence_role_present"] is True, preflight_handoff
|
|
assert preflight_handoff["submission_ref_copy_to_artifact_refs"] is True, preflight_handoff
|
|
assert preflight_handoff["supporting_evidence_copy_to_artifact_refs"] is False, preflight_handoff
|
|
assert checks["review-studio-preflight-artifact-role-handoff"]["status"] == "pass", checks[
|
|
"review-studio-preflight-artifact-role-handoff"
|
|
]
|
|
assert checks["world-class-phase-queue-consistency"]["status"] == "pass", checks[
|
|
"world-class-phase-queue-consistency"
|
|
]
|
|
phase_queue_actual = checks["world-class-phase-queue-consistency"]["actual"]
|
|
assert phase_queue_actual["summary"]["phase_queue_count"] == 2, phase_queue_actual
|
|
assert phase_queue_actual["summary"]["phase_queue_row_count"] >= 11, phase_queue_actual
|
|
assert phase_queue_actual["summary"] == phase_queue_actual["operator_runbook_summary"], phase_queue_actual
|
|
assert phase_queue_actual["top_level_phase_queue"] == phase_queue_actual[
|
|
"operator_runbook_top_level_phase_queue"
|
|
], phase_queue_actual
|
|
assert phase_queue_actual["phase_queue_counts_as_completion"] is False, phase_queue_actual
|
|
assert set(phase_queue_actual["item_phase_queues"]) == {
|
|
"provider-holdout",
|
|
"human-adjudication",
|
|
"native-permission-enforcement",
|
|
"native-client-telemetry",
|
|
}, phase_queue_actual
|
|
assert phase_queue_actual["operator_runbook_phase_queues"] == phase_queue_actual["item_phase_queues"], (
|
|
phase_queue_actual
|
|
)
|
|
assert phase_queue_actual["review_studio_phase_queues"] == phase_queue_actual["item_phase_queues"], (
|
|
phase_queue_actual
|
|
)
|
|
role_handoff = checks["review-studio-preflight-artifact-role-handoff"]["actual"]
|
|
assert role_handoff["provider-holdout"]["role_source"] == "world-class-submission-kit", role_handoff
|
|
assert role_handoff["provider-holdout"]["submission_ref_total_count"] == 1, role_handoff
|
|
assert role_handoff["provider-holdout"]["submission_ref_copy_to_artifact_refs"] is True, role_handoff
|
|
assert role_handoff["provider-holdout"]["supporting_evidence_copy_to_artifact_refs"] is False, role_handoff
|
|
assert checks["review-studio-no-overclaim"]["status"] == "pass", checks["review-studio-no-overclaim"]
|
|
assert checks["claim-guard-package-runtime-surface"]["status"] == "pass", checks[
|
|
"claim-guard-package-runtime-surface"
|
|
]
|
|
assert checks["world-class-evidence-workflow-coverage"]["status"] == "pass", checks[
|
|
"world-class-evidence-workflow-coverage"
|
|
]
|
|
workflow_actual = checks["world-class-evidence-workflow-coverage"]["actual"]
|
|
assert workflow_actual["operator_coordination_keys"] == [
|
|
"human-adjudication",
|
|
"native-client-telemetry",
|
|
"native-permission-enforcement",
|
|
"provider-holdout",
|
|
], workflow_actual
|
|
assert workflow_actual["operator_coordination_counts_as_completion"] is False, workflow_actual
|
|
assert workflow_actual["operator_release_gate_ready"] is False, workflow_actual
|
|
assert workflow_actual["operator_release_gate_counts_as_completion"] is False, workflow_actual
|
|
assert checks["skill-os-2-review-current-evidence"]["status"] == "pass", checks[
|
|
"skill-os-2-review-current-evidence"
|
|
]
|
|
markdown = output_md.read_text(encoding="utf-8")
|
|
assert "Evidence Consistency" in markdown, markdown
|
|
assert "decision: `consistent`" in markdown, markdown
|
|
assert "does not create provider, human, native-client, or permission-enforcement evidence" in markdown, markdown
|
|
|
|
drift_root = TMP / "drift-skill"
|
|
copy_reports(drift_root)
|
|
overview_path = drift_root / "reports" / "skill-overview.json"
|
|
overview = json.loads(overview_path.read_text(encoding="utf-8"))
|
|
overview["adoption_drift"]["summary"]["adoption_sample_count"] += 1
|
|
overview_path.write_text(json.dumps(overview, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(drift_root),
|
|
"--output-json",
|
|
str(TMP / "drift.json"),
|
|
"--output-md",
|
|
str(TMP / "drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert drift_proc.returncode == 2, drift_proc.stdout
|
|
drift_payload = json.loads(drift_proc.stdout)
|
|
drift_checks = {item["key"]: item for item in drift_payload["checks"]}
|
|
assert drift_payload["ok"] is False, drift_payload
|
|
assert drift_payload["summary"]["decision"] == "evidence-drift-detected", drift_payload
|
|
assert drift_checks["overview-adoption-summary"]["status"] == "fail", drift_checks["overview-adoption-summary"]
|
|
assert drift_checks["interpretation-adoption-summary"]["status"] == "pass", drift_checks["interpretation-adoption-summary"]
|
|
|
|
skill_ir_drift_root = TMP / "skill-ir-drift-skill"
|
|
copy_reports(skill_ir_drift_root)
|
|
studio_path = skill_ir_drift_root / "reports" / "review-studio.json"
|
|
studio = json.loads(studio_path.read_text(encoding="utf-8"))
|
|
studio["evidence_paths"]["skill_ir"] = "reports/skill-ir.json"
|
|
studio_path.write_text(json.dumps(studio, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
skill_ir_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(skill_ir_drift_root),
|
|
"--output-json",
|
|
str(TMP / "skill_ir_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "skill_ir_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert skill_ir_drift_proc.returncode == 2, skill_ir_drift_proc.stdout
|
|
skill_ir_drift_payload = json.loads(skill_ir_drift_proc.stdout)
|
|
skill_ir_drift_checks = {item["key"]: item for item in skill_ir_drift_payload["checks"]}
|
|
assert skill_ir_drift_checks["skill-ir-evidence-path-contract"]["status"] == "fail", (
|
|
skill_ir_drift_checks["skill-ir-evidence-path-contract"]
|
|
)
|
|
|
|
claim_guard_drift_root = TMP / "claim-guard-drift-skill"
|
|
copy_reports(claim_guard_drift_root)
|
|
claim_guard_path = claim_guard_drift_root / "reports" / "world_class_claim_guard.json"
|
|
claim_guard = json.loads(claim_guard_path.read_text(encoding="utf-8"))
|
|
claim_guard["summary"]["package_claim_surface_count"] = 0
|
|
claim_guard["scanned_surfaces"] = [
|
|
item for item in claim_guard["scanned_surfaces"] if item.get("path") != "dist/manifest.json"
|
|
]
|
|
claim_guard_path.write_text(json.dumps(claim_guard, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
claim_guard_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(claim_guard_drift_root),
|
|
"--output-json",
|
|
str(TMP / "claim_guard_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "claim_guard_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert claim_guard_drift_proc.returncode == 2, claim_guard_drift_proc.stdout
|
|
claim_guard_drift_payload = json.loads(claim_guard_drift_proc.stdout)
|
|
claim_guard_drift_checks = {item["key"]: item for item in claim_guard_drift_payload["checks"]}
|
|
assert claim_guard_drift_checks["claim-guard-package-runtime-surface"]["status"] == "fail", (
|
|
claim_guard_drift_checks["claim-guard-package-runtime-surface"]
|
|
)
|
|
|
|
preflight_drift_root = TMP / "preflight-drift-skill"
|
|
copy_reports(preflight_drift_root)
|
|
preflight_path = preflight_drift_root / "reports" / "world_class_evidence_preflight.json"
|
|
preflight = json.loads(preflight_path.read_text(encoding="utf-8"))
|
|
preflight["summary"]["credential_value_exposed"] = True
|
|
preflight_path.write_text(json.dumps(preflight, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
preflight_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(preflight_drift_root),
|
|
"--output-json",
|
|
str(TMP / "preflight_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "preflight_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert preflight_drift_proc.returncode == 2, preflight_drift_proc.stdout
|
|
preflight_drift_payload = json.loads(preflight_drift_proc.stdout)
|
|
preflight_drift_checks = {item["key"]: item for item in preflight_drift_payload["checks"]}
|
|
assert preflight_drift_checks["preflight-world-class-boundary"]["status"] == "fail", (
|
|
preflight_drift_checks["preflight-world-class-boundary"]
|
|
)
|
|
|
|
preflight_handoff_drift_root = TMP / "preflight-handoff-drift-skill"
|
|
copy_reports(preflight_handoff_drift_root)
|
|
preflight_handoff_path = preflight_handoff_drift_root / "reports" / "world_class_evidence_preflight.json"
|
|
preflight_handoff = json.loads(preflight_handoff_path.read_text(encoding="utf-8"))
|
|
preflight_handoff["submissions"]["drafts_count_as_evidence"] = True
|
|
preflight_handoff["submissions"]["commands"].pop("prepare_submission", None)
|
|
preflight_handoff_path.write_text(json.dumps(preflight_handoff, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
preflight_handoff_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(preflight_handoff_drift_root),
|
|
"--output-json",
|
|
str(TMP / "preflight_handoff_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "preflight_handoff_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert preflight_handoff_drift_proc.returncode == 2, preflight_handoff_drift_proc.stdout
|
|
preflight_handoff_drift_payload = json.loads(preflight_handoff_drift_proc.stdout)
|
|
preflight_handoff_drift_checks = {item["key"]: item for item in preflight_handoff_drift_payload["checks"]}
|
|
assert preflight_handoff_drift_checks["preflight-submission-kit-handoff"]["status"] == "fail", (
|
|
preflight_handoff_drift_checks["preflight-submission-kit-handoff"]
|
|
)
|
|
|
|
phase_queue_drift_root = TMP / "phase-queue-drift-skill"
|
|
copy_reports(phase_queue_drift_root)
|
|
phase_queue_path = phase_queue_drift_root / "reports" / "world_class_evidence_preflight.json"
|
|
phase_queue_payload = json.loads(phase_queue_path.read_text(encoding="utf-8"))
|
|
phase_queue_payload["summary"]["phase_queue_row_count"] = 1
|
|
phase_queue_payload["phase_queue"][0]["counts_as_completion"] = True
|
|
phase_queue_payload["items"][0]["phase_queue"][0]["row_count"] = 999
|
|
phase_queue_path.write_text(json.dumps(phase_queue_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
operator_phase_queue_path = phase_queue_drift_root / "reports" / "world_class_operator_runbook.json"
|
|
operator_phase_queue_payload = json.loads(operator_phase_queue_path.read_text(encoding="utf-8"))
|
|
operator_phase_queue_payload["summary"]["phase_queue_count"] = 99
|
|
operator_phase_queue_payload["phase_queue"][0]["row_count"] = 999
|
|
operator_phase_queue_payload["items"][0]["phase_queue"][0]["row_count"] = 999
|
|
operator_phase_queue_path.write_text(
|
|
json.dumps(operator_phase_queue_payload, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
phase_queue_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(phase_queue_drift_root),
|
|
"--output-json",
|
|
str(TMP / "phase_queue_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "phase_queue_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert phase_queue_drift_proc.returncode == 2, phase_queue_drift_proc.stdout
|
|
phase_queue_drift_payload = json.loads(phase_queue_drift_proc.stdout)
|
|
phase_queue_drift_checks = {item["key"]: item for item in phase_queue_drift_payload["checks"]}
|
|
assert phase_queue_drift_checks["world-class-phase-queue-consistency"]["status"] == "fail", (
|
|
phase_queue_drift_checks["world-class-phase-queue-consistency"]
|
|
)
|
|
|
|
gate_action_drift_root = TMP / "gate-action-drift-skill"
|
|
copy_reports(gate_action_drift_root)
|
|
gate_action_path = gate_action_drift_root / "reports" / "review-studio.json"
|
|
gate_action_payload = json.loads(gate_action_path.read_text(encoding="utf-8"))
|
|
for gate in gate_action_payload["gates"]:
|
|
if gate["key"] == "world-class-evidence":
|
|
gate["review_action_id"] = ""
|
|
gate["review_action_title"] = ""
|
|
gate["review_action_next_step"] = ""
|
|
gate["review_action_reason"] = ""
|
|
gate["review_action_source_ref_count"] = 0
|
|
for action in gate_action_payload["review_actions"]:
|
|
if action["gate_key"] == "world-class-evidence":
|
|
action["title"] = ""
|
|
action["next_step"] = ""
|
|
action["reason"] = ""
|
|
action["source_refs"] = []
|
|
gate_action_path.write_text(json.dumps(gate_action_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
gate_action_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(gate_action_drift_root),
|
|
"--output-json",
|
|
str(TMP / "gate_action_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "gate_action_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert gate_action_drift_proc.returncode == 2, gate_action_drift_proc.stdout
|
|
gate_action_drift_payload = json.loads(gate_action_drift_proc.stdout)
|
|
gate_action_drift_checks = {item["key"]: item for item in gate_action_drift_payload["checks"]}
|
|
assert gate_action_drift_checks["review-studio-gate-action-mirror"]["status"] == "fail", (
|
|
gate_action_drift_checks["review-studio-gate-action-mirror"]
|
|
)
|
|
|
|
release_flow_drift_root = TMP / "release-flow-drift-skill"
|
|
copy_reports(release_flow_drift_root)
|
|
agents_path = release_flow_drift_root / "AGENTS.md"
|
|
agents_text = agents_path.read_text(encoding="utf-8")
|
|
agents_path.write_text(
|
|
agents_text.replace("python3 scripts/render_skill_interpretation.py .\n", "", 1),
|
|
encoding="utf-8",
|
|
)
|
|
release_flow_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(release_flow_drift_root),
|
|
"--output-json",
|
|
str(TMP / "release_flow_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "release_flow_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert release_flow_drift_proc.returncode == 2, release_flow_drift_proc.stdout
|
|
release_flow_drift_payload = json.loads(release_flow_drift_proc.stdout)
|
|
release_flow_drift_checks = {item["key"]: item for item in release_flow_drift_payload["checks"]}
|
|
assert release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]["status"] == "fail", (
|
|
release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]
|
|
)
|
|
|
|
context_drift_root = TMP / "context-drift-skill"
|
|
copy_reports(context_drift_root)
|
|
context_path = context_drift_root / "reports" / "context_budget.json"
|
|
context_payload = json.loads(context_path.read_text(encoding="utf-8"))
|
|
context_payload["warnings"] = ["Deferred resource footprint is high: stale test warning."]
|
|
context_payload["stats"]["deferred_resource_tokens"] = 1
|
|
context_payload["stats"]["deferred_resource_governance"]["status"] = "pass"
|
|
context_path.write_text(json.dumps(context_payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
context_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(context_drift_root),
|
|
"--output-json",
|
|
str(TMP / "context_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "context_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert context_drift_proc.returncode == 2, context_drift_proc.stdout
|
|
context_drift_payload = json.loads(context_drift_proc.stdout)
|
|
context_drift_checks = {item["key"]: item for item in context_drift_payload["checks"]}
|
|
assert context_drift_checks["review-studio-context-budget-mirror"]["status"] == "fail", (
|
|
context_drift_checks["review-studio-context-budget-mirror"]
|
|
)
|
|
|
|
benchmark_flow_drift_root = TMP / "benchmark-flow-drift-skill"
|
|
copy_reports(benchmark_flow_drift_root)
|
|
agents_path = benchmark_flow_drift_root / "AGENTS.md"
|
|
agents_text = agents_path.read_text(encoding="utf-8")
|
|
clean_lock_header = "For final release evidence"
|
|
benchmark_command = 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"\n'
|
|
prefix, suffix = agents_text.split(clean_lock_header, 1)
|
|
agents_path.write_text(
|
|
prefix + clean_lock_header + suffix.replace(benchmark_command, "", 1),
|
|
encoding="utf-8",
|
|
)
|
|
benchmark_flow_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(benchmark_flow_drift_root),
|
|
"--output-json",
|
|
str(TMP / "benchmark_flow_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "benchmark_flow_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert benchmark_flow_drift_proc.returncode == 2, benchmark_flow_drift_proc.stdout
|
|
benchmark_flow_drift_payload = json.loads(benchmark_flow_drift_proc.stdout)
|
|
benchmark_flow_drift_checks = {item["key"]: item for item in benchmark_flow_drift_payload["checks"]}
|
|
assert benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]["status"] == "fail", (
|
|
benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]
|
|
)
|
|
|
|
stale_clean_lock_root = TMP / "stale-clean-lock-skill"
|
|
copy_reports(stale_clean_lock_root)
|
|
benchmark_path = stale_clean_lock_root / "reports" / "benchmark_reproducibility.json"
|
|
benchmark = json.loads(benchmark_path.read_text(encoding="utf-8"))
|
|
benchmark["git_status"]["dirty"] = True
|
|
benchmark["git_status"]["changed_file_count"] = 3
|
|
benchmark["git_status"]["source_dirty"] = True
|
|
benchmark["git_status"]["source_changed_file_count"] = 1
|
|
benchmark["git_status"]["generated_dirty"] = True
|
|
benchmark["git_status"]["generated_changed_file_count"] = 2
|
|
benchmark["summary"]["release_lock_ready"] = False
|
|
benchmark["summary"]["working_tree_dirty"] = True
|
|
benchmark["summary"]["changed_file_count"] = 3
|
|
benchmark["summary"]["source_tree_dirty"] = True
|
|
benchmark["summary"]["source_changed_file_count"] = 1
|
|
benchmark["summary"]["generated_tree_dirty"] = True
|
|
benchmark["summary"]["generated_changed_file_count"] = 2
|
|
benchmark["release_lock"]["ready"] = False
|
|
benchmark["release_lock"]["reason"] = "source files were dirty at generation time"
|
|
benchmark_path.write_text(json.dumps(benchmark, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
for report_name in ["skill-overview", "skill-interpretation"]:
|
|
report_path = stale_clean_lock_root / "reports" / f"{report_name}.json"
|
|
report = json.loads(report_path.read_text(encoding="utf-8"))
|
|
report["benchmark_reproducibility"]["summary"]["release_lock_ready"] = False
|
|
report["benchmark_reproducibility"]["summary"]["working_tree_dirty"] = True
|
|
report["benchmark_reproducibility"]["summary"]["changed_file_count"] = 3
|
|
report["benchmark_reproducibility"]["summary"]["source_tree_dirty"] = True
|
|
report["benchmark_reproducibility"]["summary"]["source_changed_file_count"] = 1
|
|
report["benchmark_reproducibility"]["summary"]["generated_tree_dirty"] = True
|
|
report["benchmark_reproducibility"]["summary"]["generated_changed_file_count"] = 2
|
|
report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
subprocess.run(["git", "init"], cwd=stale_clean_lock_root, capture_output=True, text=True, check=True)
|
|
subprocess.run(["git", "add", "."], cwd=stale_clean_lock_root, capture_output=True, text=True, check=True)
|
|
subprocess.run(
|
|
["git", "-c", "user.name=Yao Test", "-c", "user.email=yao-test@example.com", "commit", "-m", "seed"],
|
|
cwd=stale_clean_lock_root,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
stale_clean_lock_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(stale_clean_lock_root),
|
|
"--output-json",
|
|
str(TMP / "stale_clean_lock.json"),
|
|
"--output-md",
|
|
str(TMP / "stale_clean_lock.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert stale_clean_lock_proc.returncode == 2, stale_clean_lock_proc.stdout
|
|
stale_clean_lock_payload = json.loads(stale_clean_lock_proc.stdout)
|
|
stale_clean_lock_checks = {item["key"]: item for item in stale_clean_lock_payload["checks"]}
|
|
assert stale_clean_lock_checks["benchmark-release-lock-self-consistency"]["status"] == "pass", (
|
|
stale_clean_lock_checks["benchmark-release-lock-self-consistency"]
|
|
)
|
|
assert stale_clean_lock_checks["benchmark-clean-worktree-release-lock"]["status"] == "fail", (
|
|
stale_clean_lock_checks["benchmark-clean-worktree-release-lock"]
|
|
)
|
|
|
|
generated_only_lock_root = TMP / "generated-only-lock-skill"
|
|
copy_reports(generated_only_lock_root)
|
|
generated_benchmark_path = generated_only_lock_root / "reports" / "benchmark_reproducibility.json"
|
|
generated_benchmark = json.loads(generated_benchmark_path.read_text(encoding="utf-8"))
|
|
generated_benchmark["git_status"]["dirty"] = True
|
|
generated_benchmark["git_status"]["changed_file_count"] = 4
|
|
generated_benchmark["git_status"]["source_dirty"] = False
|
|
generated_benchmark["git_status"]["source_changed_file_count"] = 0
|
|
generated_benchmark["git_status"]["generated_dirty"] = True
|
|
generated_benchmark["git_status"]["generated_changed_file_count"] = 4
|
|
generated_benchmark["summary"]["release_lock_ready"] = True
|
|
generated_benchmark["summary"]["working_tree_dirty"] = True
|
|
generated_benchmark["summary"]["changed_file_count"] = 4
|
|
generated_benchmark["summary"]["source_tree_dirty"] = False
|
|
generated_benchmark["summary"]["source_changed_file_count"] = 0
|
|
generated_benchmark["summary"]["generated_tree_dirty"] = True
|
|
generated_benchmark["summary"]["generated_changed_file_count"] = 4
|
|
generated_benchmark["release_lock"]["ready"] = True
|
|
generated_benchmark["release_lock"]["reason"] = "only generated evidence artifacts were dirty at generation time"
|
|
generated_benchmark_path.write_text(
|
|
json.dumps(generated_benchmark, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
generated_only_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(generated_only_lock_root),
|
|
"--output-json",
|
|
str(TMP / "generated_only_lock.json"),
|
|
"--output-md",
|
|
str(TMP / "generated_only_lock.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
generated_only_payload = json.loads(generated_only_proc.stdout)
|
|
generated_only_checks = {item["key"]: item for item in generated_only_payload["checks"]}
|
|
assert generated_only_checks["benchmark-release-lock-self-consistency"]["status"] == "pass", (
|
|
generated_only_checks["benchmark-release-lock-self-consistency"]
|
|
)
|
|
|
|
workflow_drift_root = TMP / "workflow-drift-skill"
|
|
copy_reports(workflow_drift_root)
|
|
studio_path = workflow_drift_root / "reports" / "review-studio.json"
|
|
studio = json.loads(studio_path.read_text(encoding="utf-8"))
|
|
for action in studio["review_actions"]:
|
|
if action.get("gate_key") == "world-class-evidence":
|
|
action["evidence_steps"] = [
|
|
item for item in action["evidence_steps"] if item.get("key") != "native-client-telemetry"
|
|
]
|
|
studio_path.write_text(json.dumps(studio, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
operator_runbook_path = workflow_drift_root / "reports" / "world_class_operator_runbook.json"
|
|
operator_runbook = json.loads(operator_runbook_path.read_text(encoding="utf-8"))
|
|
operator_runbook["coordination_plan"] = [
|
|
item
|
|
for item in operator_runbook["coordination_plan"]
|
|
if item.get("evidence_key") != "native-client-telemetry"
|
|
]
|
|
operator_runbook["summary"]["coordination_counts_as_completion"] = True
|
|
operator_runbook["release_gate"]["counts_as_completion"] = True
|
|
operator_runbook_path.write_text(json.dumps(operator_runbook, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
|
workflow_drift_proc = run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(workflow_drift_root),
|
|
"--output-json",
|
|
str(TMP / "workflow_drift.json"),
|
|
"--output-md",
|
|
str(TMP / "workflow_drift.md"),
|
|
"--generated-at",
|
|
"2026-06-15",
|
|
]
|
|
)
|
|
assert workflow_drift_proc.returncode == 2, workflow_drift_proc.stdout
|
|
workflow_drift_payload = json.loads(workflow_drift_proc.stdout)
|
|
workflow_drift_checks = {item["key"]: item for item in workflow_drift_payload["checks"]}
|
|
assert workflow_drift_checks["world-class-evidence-workflow-coverage"]["status"] == "fail", (
|
|
workflow_drift_checks["world-class-evidence-workflow-coverage"]
|
|
)
|
|
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|