yao-meta-skill/tests/verify_world_class_preflight.py

#!/usr/bin/env python3
import json
import os
import shutil
import subprocess
import sys
from pathlib import Path


ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "render_world_class_preflight.py"
CLI = ROOT / "scripts" / "yao.py"
TMP = ROOT / "tests" / "tmp_world_class_preflight"


def run_preflight(extra_env: dict[str, str] | None = None, *extra: str) -> subprocess.CompletedProcess[str]:
    env = dict(os.environ)
    env.pop("OPENAI_API_KEY", None)
    env.pop("YAO_OUTPUT_EVAL_MODEL", None)
    if extra_env:
        env.update(extra_env)
    return subprocess.run(
        [
            sys.executable,
            str(SCRIPT),
            str(ROOT),
            "--generated-at",
            "2026-06-16",
            *extra,
        ],
        cwd=ROOT,
        capture_output=True,
        text=True,
        env=env,
        check=True,
    )


def run_cli(*extra: str) -> subprocess.CompletedProcess[str]:
    env = dict(os.environ)
    env["YAO_CLI_TELEMETRY"] = "0"
    env.pop("OPENAI_API_KEY", None)
    return subprocess.run(
        [sys.executable, str(CLI), "world-class-preflight", str(ROOT), *extra],
        cwd=ROOT,
        capture_output=True,
        text=True,
        env=env,
        check=True,
    )


def by_key(items: list[dict], key: str) -> dict:
    return next(item for item in items if item.get("evidence_key") == key)


def main() -> None:
    shutil.rmtree(TMP, ignore_errors=True)
    TMP.mkdir(parents=True, exist_ok=True)

    output_json = TMP / "world_class_evidence_preflight.json"
    output_md = TMP / "world_class_evidence_preflight.md"
    output_html = TMP / "world_class_evidence_preflight.html"
    proc = run_preflight(None, "--output-json", str(output_json), "--output-md", str(output_md), "--output-html", str(output_html))
    payload = json.loads(proc.stdout)
    assert payload["schema_version"] == "1.0", payload
    assert payload["ok"] is True, payload
    summary = payload["summary"]
    assert summary["decision"] == "collection-preflight-blocked", summary
    assert summary["ready_to_claim_world_class"] is False, summary
    assert summary["preflight_counts_as_evidence"] is False, summary
    assert summary["credential_value_exposed"] is False, summary
    assert summary["evidence_item_count"] == 4, summary
    assert summary["pending_count"] == 4, summary
    assert summary["precheck_count"] >= 12, summary
    assert summary["precheck_missing_count"] >= 1, summary
    assert summary["precheck_external_required_count"] == 2, summary
    assert summary["precheck_human_required_count"] == 1, summary
    assert summary["source_check_count"] >= 13, summary
    assert summary["source_pass_count"] + summary["source_blocked_count"] == summary["source_check_count"], summary
    assert summary["source_blocked_count"] >= 6, summary
    assert summary["repair_checklist_count"] == (
        summary["source_blocked_count"]
        + summary["precheck_missing_count"]
        + summary["precheck_human_required_count"]
        + summary["precheck_external_required_count"]
    ), summary
    assert summary["repair_blocked_count"] == summary["repair_checklist_count"], summary
    assert summary["repair_ready_count"] == 0, summary
    assert summary["repair_counts_as_completion"] is False, summary
    assert payload["submissions"]["preflight_counts_submission_as_completion"] is False, payload
    assert payload["submissions"]["drafts_count_as_evidence"] is False, payload
    assert payload["submissions"]["submission_kit_command"] == (
        "python3 scripts/yao.py world-class-submission-kit . "
        "--output-dir evidence/world_class/submissions"
    ), payload["submissions"]
    assert payload["submissions"]["submission_kit_prefill_command"] == (
        "python3 scripts/yao.py world-class-submission-kit . "
        "--output-dir evidence/world_class/submissions --prefill-artifacts"
    ), payload["submissions"]
    assert payload["submissions"]["artifact_prefill_counts_as_evidence"] is False, payload
    role_contract = payload["submissions"]["artifact_role_contract"]
    assert role_contract["role_source"] == "world-class-submission-kit", role_contract
    assert role_contract["counts_as_evidence"] is False, role_contract
    assert role_contract["artifact_prefill_counts_as_evidence"] is False, role_contract
    assert role_contract["submission_ref_total_count"] >= 4, role_contract
    assert role_contract["submission_ref_ready_count"] <= role_contract["submission_ref_total_count"], role_contract
    assert role_contract["supporting_evidence_total_count"] >= 4, role_contract
    roles = {item["role"]: item for item in role_contract["roles"]}
    assert roles["submission-ref"]["copy_to_artifact_refs"] is True, roles
    assert roles["supporting-evidence"]["copy_to_artifact_refs"] is False, roles
    assert payload["artifacts"]["html"] == "reports/world_class_evidence_preflight.html", payload["artifacts"]
    submission_commands = payload["submissions"]["commands"]
    assert submission_commands["prepare_submission"] == (
        "python3 scripts/yao.py world-class-submission-kit . "
        "--output-dir evidence/world_class/submissions"
    ), submission_commands
    assert submission_commands["prepare_prefilled_submission"] == (
        "python3 scripts/yao.py world-class-submission-kit . "
        "--output-dir evidence/world_class/submissions --prefill-artifacts"
    ), submission_commands
    assert submission_commands["validate_intake"] == (
        "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions"
    ), submission_commands
    assert submission_commands["submission_review"] == (
        "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions"
    ), submission_commands
    assert submission_commands["refresh_ledger"] == (
        "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions"
    ), submission_commands
    assert submission_commands["guard_claim"] == "python3 scripts/yao.py world-class-claim-guard .", submission_commands

    provider = by_key(payload["items"], "provider-holdout")
    assert provider["status"] == "blocked", provider
    assert provider["commands"]["prepare_submission"] == (
        "python3 scripts/yao.py world-class-submission-kit . "
        "--evidence-key provider-holdout --output-dir evidence/world_class/submissions"
    ), provider
    assert provider["commands"]["prepare_prefilled_submission"] == (
        "python3 scripts/yao.py world-class-submission-kit . "
        "--evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts"
    ), provider
    assert provider["submission_kit"]["drafts_count_as_evidence"] is False, provider
    assert provider["submission_kit"]["artifact_prefill_counts_as_evidence"] is False, provider
    assert provider["submission_kit"]["prefill_command"] == provider["commands"]["prepare_prefilled_submission"], provider
    assert provider["submission_kit"]["output_dir"] == "evidence/world_class/submissions", provider
    assert provider["submission_kit"]["draft_path"] == "evidence/world_class/submissions/provider-holdout.json", provider
    provider_roles = provider["submission_kit"]["artifact_role_contract"]
    assert provider_roles["submission_ref_total_count"] == 1, provider_roles
    assert provider_roles["submission_ref_ready_count"] == 1, provider_roles
    assert provider_roles["supporting_evidence_total_count"] >= 1, provider_roles
    provider_checks = {item["key"]: item for item in provider["prechecks"]}
    assert provider_checks["openai-api-key"]["status"] == "missing", provider_checks
    assert provider_checks["openai-api-key"]["actual"] == "not-set", provider_checks
    assert provider_checks["openai-api-key"]["secret_value_redacted"] is True, provider_checks
    assert "sk-test-secret" not in proc.stdout, proc.stdout
    assert "OPENAI_API_KEY" in proc.stdout, proc.stdout
    assert "set OPENAI_API_KEY" not in proc.stdout.lower(), proc.stdout
    provider_repairs = {item["target"]: item for item in provider["repair_checklist"]}
    assert provider_repairs["openai-api-key"]["repair_type"] == "precheck", provider_repairs
    assert provider_repairs["openai-api-key"]["counts_as_completion"] is False, provider_repairs
    assert provider_repairs["model_executed_count"]["repair_type"] == "source-check", provider_repairs
    assert provider_repairs["token_observed_count"]["status"] == "blocked", provider_repairs

    human = by_key(payload["items"], "human-adjudication")
    assert human["status"] == "ready-for-human-review", human
    human_checks = {item["key"]: item for item in human["prechecks"]}
    assert human_checks["human-reviewer"]["status"] == "human-required", human_checks
    assert "reviewer identity" in human["next_action"], human
    assert any("Record a reviewer choice and reason" in row["next_action"] for row in human["source_checklist"]), human
    assert any("required rationale" in item["next_action"] for item in human["prechecks"]), human
    assert any("reviewed_at" in item["next_action"] for item in human["prechecks"]), human
    human_repairs = {item["target"]: item for item in human["repair_checklist"]}
    assert human_repairs["human-reviewer"]["repair_type"] == "precheck", human_repairs
    assert human_repairs["pending_count"]["repair_type"] == "source-check", human_repairs

    native = by_key(payload["items"], "native-permission-enforcement")
    assert native["status"] == "blocked", native
    assert any(item["status"] == "external-required" for item in native["prechecks"]), native

    markdown = output_md.read_text(encoding="utf-8")
    assert "World-Class Evidence Preflight" in markdown, markdown
    assert "ready to claim world-class: `false`" in markdown, markdown
    assert "preflight counts as evidence: `false`" in markdown, markdown
    assert "credential value exposed: `false`" in markdown, markdown
    assert "Submission Kit Handoff" in markdown, markdown
    assert "Repair Checklist" in markdown, markdown
    assert "Repair rows convert preflight and source blockers" in markdown, markdown
    assert "repair rows:" in markdown, markdown
    assert "`openai-api-key`" in markdown, markdown
    assert "`model_executed_count`" in markdown, markdown
    assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in markdown, markdown
    assert "world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts" in markdown, markdown
    assert "world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions" in markdown, markdown
    assert "world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions --prefill-artifacts" in markdown, markdown
    assert "drafts count as evidence: `false`" in markdown, markdown
    assert "artifact prefill counts as evidence: `false`" in markdown, markdown
    assert "does not make a draft count as evidence" in markdown, markdown
    assert "submission refs ready:" in markdown, markdown
    assert "`submission-ref` rows are the only checklist rows expected in `artifact_refs`" in markdown, markdown
    assert "| `supporting-evidence` | `false` |" in markdown, markdown
    assert "values are never printed" in markdown, markdown
    html = output_html.read_text(encoding="utf-8")
    assert "<title>World-Class Evidence Preflight</title>" in html, html
    assert "World-Class Evidence Preflight" in html, html
    assert "Evidence Queue" in html, html
    assert "Submission Kit" in html, html
    assert "Repair Rows" in html, html
    assert "Repairs" in html, html
    assert "repair-row blocked" in html, html
    assert "<strong>openai-api-key</strong>" in html, html
    assert "<strong>model_executed_count</strong>" in html, html
    assert "world-class-submission-kit . --output-dir evidence/world_class/submissions" in html, html
    assert "world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifacts" in html, html
    assert "provider-holdout" in html, html
    assert "Artifact prefill is convenience data only." in html, html
    assert "artifact prefill counts as evidence" in html, html
    assert "Artifact Roles" in html, html
    assert "copy to artifact_refs: <code>true</code>" in html, html
    assert "supporting-evidence" in html, html
    assert "Source Checks" in html, html
    assert "ready_to_claim_world_class" in html, html
    assert "Environment variables are displayed only as set or not-set" in html, html

    env_json = TMP / "preflight_with_env.json"
    env_proc = run_preflight(
        {"OPENAI_API_KEY": "sk-test-secret", "YAO_OUTPUT_EVAL_MODEL": "gpt-test"},
        "--output-json",
        str(env_json),
        "--output-md",
        str(TMP / "preflight_with_env.md"),
        "--output-html",
        str(TMP / "preflight_with_env.html"),
    )
    env_payload = json.loads(env_proc.stdout)
    env_provider = by_key(env_payload["items"], "provider-holdout")
    env_provider_checks = {item["key"]: item for item in env_provider["prechecks"]}
    assert env_provider_checks["openai-api-key"]["status"] == "pass", env_provider_checks
    assert env_provider_checks["openai-api-key"]["actual"] == "set", env_provider_checks
    assert env_provider_checks["provider-model"]["status"] == "pass", env_provider_checks
    assert "sk-test-secret" not in env_proc.stdout, env_proc.stdout
    assert "sk-test-secret" not in (TMP / "preflight_with_env.html").read_text(encoding="utf-8"), env_payload
    assert env_payload["summary"]["credential_value_exposed"] is False, env_payload
    assert env_payload["summary"]["ready_to_claim_world_class"] is False, env_payload
    assert env_payload["summary"]["repair_checklist_count"] == payload["summary"]["repair_checklist_count"] - 1, env_payload

    spaced_dir = TMP / "submission kit spaced"
    spaced_proc = run_preflight(
        None,
        "--submissions-dir",
        str(spaced_dir),
        "--output-json",
        str(TMP / "preflight_spaced.json"),
        "--output-md",
        str(TMP / "preflight_spaced.md"),
        "--output-html",
        str(TMP / "preflight_spaced.html"),
    )
    spaced_payload = json.loads(spaced_proc.stdout)
    quoted_spaced = "'tests/tmp_world_class_preflight/submission kit spaced'"
    assert quoted_spaced in spaced_payload["submissions"]["commands"]["prepare_submission"], spaced_payload["submissions"]
    assert quoted_spaced in spaced_payload["submissions"]["commands"]["prepare_prefilled_submission"], spaced_payload["submissions"]
    assert quoted_spaced in by_key(spaced_payload["items"], "provider-holdout")["commands"]["prepare_submission"], spaced_payload["items"]
    assert quoted_spaced in by_key(spaced_payload["items"], "provider-holdout")["commands"]["prepare_prefilled_submission"], spaced_payload["items"]

    cli_proc = run_cli(
        "--output-json",
        str(TMP / "cli_preflight.json"),
        "--output-md",
        str(TMP / "cli_preflight.md"),
        "--output-html",
        str(TMP / "cli_preflight.html"),
        "--generated-at",
        "2026-06-16",
    )
    cli_payload = json.loads(cli_proc.stdout)
    assert cli_payload["summary"]["decision"] == "collection-preflight-blocked", cli_payload
    assert cli_payload["summary"]["preflight_counts_as_evidence"] is False, cli_payload
    assert (TMP / "cli_preflight.md").exists(), cli_payload
    assert (TMP / "cli_preflight.html").exists(), cli_payload

    print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))


if __name__ == "__main__":
    main()