yaojingang
368 lines
23 KiB
Python
368 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
import hashlib
|
|
import json
|
|
import shutil
|
|
import subprocess
|
|
import sys
|
|
from pathlib import Path
|
|
|
|
|
|
ROOT = Path(__file__).resolve().parent.parent
|
|
SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py"
|
|
KIT_SCRIPT = ROOT / "scripts" / "prepare_world_class_submission_kit.py"
|
|
TMP = ROOT / "tests" / "tmp_world_class_evidence_intake"
|
|
|
|
|
|
def sha256_file(path: Path) -> str:
|
|
digest = hashlib.sha256()
|
|
with path.open("rb") as handle:
|
|
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
|
|
digest.update(chunk)
|
|
return digest.hexdigest()
|
|
|
|
|
|
def run_intake(*extra: str) -> dict:
|
|
proc = subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(SCRIPT),
|
|
str(ROOT),
|
|
"--generated-at",
|
|
"2026-06-14",
|
|
*extra,
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
return json.loads(proc.stdout)
|
|
|
|
|
|
def run_kit(*extra: str) -> subprocess.CompletedProcess[str]:
|
|
return subprocess.run(
|
|
[
|
|
sys.executable,
|
|
str(KIT_SCRIPT),
|
|
str(ROOT),
|
|
"--generated-at",
|
|
"2026-06-14",
|
|
*extra,
|
|
],
|
|
cwd=ROOT,
|
|
capture_output=True,
|
|
text=True,
|
|
check=True,
|
|
)
|
|
|
|
|
|
def provider_submission(*, valid: bool, artifact_path: str = "reports/output_execution_runs.json") -> dict:
|
|
return {
|
|
"schema_version": "1.0",
|
|
"evidence_key": "provider-holdout",
|
|
"template_only": False,
|
|
"category": "external",
|
|
"source_type": "provider-output-eval",
|
|
"submitted_by": "Yao provider operator",
|
|
"submitted_at": "2026-06-14",
|
|
"summary": "Aggregate provider-backed holdout evidence for ledger review.",
|
|
"artifact_refs": [
|
|
{
|
|
"path": artifact_path,
|
|
"kind": "aggregate-report",
|
|
"contains_raw_content": not valid,
|
|
"sha256": sha256_file(ROOT / artifact_path) if valid else "example-only",
|
|
}
|
|
],
|
|
"provenance": {
|
|
"provider": "openai",
|
|
"model": "gpt-4.1-mini",
|
|
"credential_material_committed": False,
|
|
},
|
|
"privacy": {
|
|
"raw_user_content_included": False,
|
|
"raw_provider_prompt_included": False,
|
|
"credentials_included": False,
|
|
"secrets_included": False,
|
|
},
|
|
"anti_overclaim": {
|
|
"planned_work_counts_as_evidence": False,
|
|
"metadata_fallback_counts_as_native_enforcement": False,
|
|
"pending_review_counts_as_human_decision": False,
|
|
"local_command_runner_counts_as_provider_model": False,
|
|
},
|
|
"attestation": {
|
|
"real_external_or_human_evidence": valid,
|
|
"reviewer_or_operator_identity_present": valid,
|
|
"artifact_refs_reviewed": valid,
|
|
"privacy_contract_satisfied": valid,
|
|
},
|
|
}
|
|
|
|
|
|
def assert_documented_submission_commands() -> None:
|
|
expected_fragments = [
|
|
'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"',
|
|
'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR"',
|
|
'world-class-intake . --submissions-dir "$SUBMISSIONS_DIR"',
|
|
'world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR"',
|
|
'world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR"',
|
|
]
|
|
for relative_path in ("README.md", "evidence/world_class/README.md"):
|
|
text = (ROOT / relative_path).read_text(encoding="utf-8")
|
|
for fragment in expected_fragments:
|
|
assert fragment in text, (relative_path, fragment)
|
|
assert "/tmp/yao-world-class-submission-kit" not in text, relative_path
|
|
|
|
|
|
def main() -> None:
|
|
shutil.rmtree(TMP, ignore_errors=True)
|
|
TMP.mkdir(parents=True, exist_ok=True)
|
|
assert_documented_submission_commands()
|
|
default_json = TMP / "world_class_evidence_intake.json"
|
|
default_md = TMP / "world_class_evidence_intake.md"
|
|
payload = run_intake("--output-json", str(default_json), "--output-md", str(default_md))
|
|
assert payload["schema_version"] == "1.0", payload
|
|
assert payload["ok"] is True, payload
|
|
summary = payload["summary"]
|
|
assert summary["decision"] == "awaiting-submissions", summary
|
|
assert summary["schema_present"] is True, summary
|
|
assert summary["ledger_entry_count"] == 4, summary
|
|
assert summary["template_count"] == 4, summary
|
|
assert summary["template_pass_count"] == 4, summary
|
|
assert summary["submission_count"] == 0, summary
|
|
assert summary["valid_submission_count"] == 0, summary
|
|
assert summary["invalid_submission_count"] == 0, summary
|
|
assert summary["valid_packet_source_incomplete_count"] == 0, summary
|
|
assert summary["operator_checklist_count"] == 4, summary
|
|
assert summary["operator_checklist_ready_count"] == 0, summary
|
|
assert summary["ready_for_external_collection"] is True, summary
|
|
assert summary["ready_for_ledger_review"] is False, summary
|
|
assert summary["ready_to_claim_world_class"] is False, summary
|
|
assert summary["overclaim_guard_active"] is True, summary
|
|
assert {item["evidence_key"] for item in payload["templates"]} == {
|
|
"provider-holdout",
|
|
"human-adjudication",
|
|
"native-permission-enforcement",
|
|
"native-client-telemetry",
|
|
}, payload["templates"]
|
|
assert all(item["status"] == "pass" and item["template_only"] is True for item in payload["templates"]), payload["templates"]
|
|
checklist = {item["evidence_key"]: item for item in payload["operator_checklist"]}
|
|
assert set(checklist) == {
|
|
"provider-holdout",
|
|
"human-adjudication",
|
|
"native-permission-enforcement",
|
|
"native-client-telemetry",
|
|
}, checklist
|
|
assert checklist["provider-holdout"]["readiness"] == "awaiting-submission", checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["template_path"] == "evidence/world_class/templates/provider-holdout.intake.json", checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["submission_path"] == "evidence/world_class/submissions/provider-holdout.json", checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["commands"]["prepare_submission"] == (
|
|
"python3 scripts/yao.py world-class-submission-kit . "
|
|
"--evidence-key provider-holdout --output-dir evidence/world_class/submissions"
|
|
), checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["commands"]["validate_intake"] == "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["commands"]["submission_review"] == "python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["commands"]["refresh_ledger"] == "python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions", checklist["provider-holdout"]
|
|
assert "provider-backed model run" in checklist["provider-holdout"]["must_collect"]["provenance_requirements"], checklist["provider-holdout"]
|
|
assert any("output-exec --provider-runner openai" in step for step in checklist["provider-holdout"]["must_collect"]["runbook"]), checklist["provider-holdout"]
|
|
assert "reports/output_execution_runs.json summary.model_executed_count > 0" in checklist["provider-holdout"]["must_collect"]["success_checks"], checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["anti_overclaim"]["local_command_runner_counts_as_provider_model"] is False, checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["source_accepted"] is False, checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["observed_state"]["model_executed_count"] == 0, checklist["provider-holdout"]
|
|
assert checklist["provider-holdout"]["observed_state"]["timing_observed_count"] > 0, checklist["provider-holdout"]
|
|
markdown = default_md.read_text(encoding="utf-8")
|
|
assert "World-Class Evidence Intake" in markdown, markdown
|
|
assert "ready to claim world-class: `false`" in markdown, markdown
|
|
assert "Operator Checklist" in markdown, markdown
|
|
assert "valid packet but source incomplete: `0`" in markdown, markdown
|
|
assert "operator checklist: `0` ready / `4` total" in markdown, markdown
|
|
assert "0 existing / 0 sha256 verified / 0 required verified / 1 refs" in markdown, markdown
|
|
assert "`evidence/world_class/submissions/provider-holdout.json`" in markdown, markdown
|
|
assert "`python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions`" in markdown, markdown
|
|
assert "`python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions`" in markdown, markdown
|
|
assert "`python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions`" in markdown, markdown
|
|
assert "Source Runbook" in markdown, markdown
|
|
assert "output-exec --provider-runner openai" in markdown, markdown
|
|
assert "Templates and planned work do not count as accepted evidence." in markdown, markdown
|
|
assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown
|
|
assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown
|
|
|
|
kit_dir = TMP / "submission_kit"
|
|
kit_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout")
|
|
kit_payload = json.loads(kit_proc.stdout)
|
|
assert kit_payload["ok"] is True, kit_payload
|
|
assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"]
|
|
assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"]
|
|
assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"]
|
|
assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"]
|
|
assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"]
|
|
assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"]
|
|
assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"]
|
|
assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"]
|
|
assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"]
|
|
assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"]
|
|
assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"]
|
|
assert kit_payload["files"][0]["output_path"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/provider-holdout.json"), kit_payload["files"]
|
|
artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]}
|
|
assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows
|
|
assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows
|
|
assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows
|
|
assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows
|
|
source_rows = {item["field"]: item for item in kit_payload["source_checklist"]}
|
|
assert source_rows["model_executed_count"]["status"] == "blocked", source_rows
|
|
assert source_rows["model_executed_count"]["actual"] == 0, source_rows
|
|
assert source_rows["timing_observed_count"]["status"] == "pass", source_rows
|
|
assert source_rows["token_observed_count"]["status"] == "blocked", source_rows
|
|
kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8"))
|
|
assert kit_draft["template_only"] is True, kit_draft
|
|
assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft
|
|
kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8"))
|
|
assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"]
|
|
assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"]
|
|
assert kit_manifest["source_checklist"] == kit_payload["source_checklist"], kit_manifest["source_checklist"]
|
|
assert kit_manifest["artifacts"]["html"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/index.html"), kit_manifest["artifacts"]
|
|
kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8")
|
|
assert "Drafts are not accepted evidence." in kit_readme, kit_readme
|
|
assert "Execution Runbook" in kit_readme, kit_readme
|
|
assert "output-exec --provider-runner openai" in kit_readme, kit_readme
|
|
assert "validate intake" in kit_readme, kit_readme
|
|
assert "Artifact Checklist" in kit_readme, kit_readme
|
|
assert "Source Evidence Snapshot" in kit_readme, kit_readme
|
|
assert "reports/output_execution_runs.json" in kit_readme, kit_readme
|
|
assert "Provider model run" in kit_readme, kit_readme
|
|
kit_html = (kit_dir / "index.html").read_text(encoding="utf-8")
|
|
assert "<title>World-Class Evidence Submission Kit</title>" in kit_html, kit_html
|
|
assert "Drafts are not accepted evidence" in kit_html, kit_html
|
|
assert "provider-holdout" in kit_html, kit_html
|
|
assert "Artifact Checklist" in kit_html, kit_html
|
|
assert "Source Evidence Snapshot" in kit_html, kit_html
|
|
assert "<dt>Field</dt><dd><code>model_executed_count</code></dd>" in kit_html, kit_html
|
|
assert "<dt>Current</dt><dd><code>0</code></dd>" in kit_html, kit_html
|
|
assert "<dt>Expected</dt><dd><code>>0</code></dd>" in kit_html, kit_html
|
|
assert "World-Class Evidence Submission Kit" in kit_html, kit_html
|
|
assert "Execution Runbook" in kit_html, kit_html
|
|
assert "output-exec --provider-runner openai" in kit_html, kit_html
|
|
assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html
|
|
|
|
native_kit_dir = TMP / "native_permission_kit"
|
|
native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement")
|
|
native_kit_payload = json.loads(native_kit_proc.stdout)
|
|
native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]}
|
|
assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 1, native_kit_payload["summary"]
|
|
if "dist/targets/openai/adapter.json" in native_rows:
|
|
assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows
|
|
assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows
|
|
assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows
|
|
else:
|
|
glob_rows = [
|
|
item for item in native_kit_payload["artifact_checklist"]
|
|
if item["source_pattern"] == "dist/targets/*/adapter.json"
|
|
]
|
|
assert len(glob_rows) == 1, glob_rows
|
|
assert glob_rows[0]["status"] == "glob-no-match", glob_rows[0]
|
|
assert glob_rows[0]["artifact_ref_ready"] is False, glob_rows[0]
|
|
assert glob_rows[0]["concrete_reference_required"] is True, glob_rows[0]
|
|
native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8")
|
|
assert "Glob patterns are expanded into concrete files" in native_readme, native_readme
|
|
draft_intake = run_intake("--submissions-dir", str(kit_dir))
|
|
assert draft_intake["ok"] is False, draft_intake
|
|
assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"]
|
|
assert draft_intake["summary"]["invalid_submission_count"] == 1, draft_intake["summary"]
|
|
assert draft_intake["submissions"][0]["evidence_key"] == "provider-holdout", draft_intake["submissions"]
|
|
assert all(item["evidence_key"] != "unknown" for item in draft_intake["submissions"]), draft_intake["submissions"]
|
|
|
|
spaced_kit_dir = TMP / "submission kit spaced"
|
|
spaced_kit_proc = run_kit("--output-dir", str(spaced_kit_dir), "--evidence-key", "provider-holdout")
|
|
spaced_kit_payload = json.loads(spaced_kit_proc.stdout)
|
|
quoted_spaced_dir = "'tests/tmp_world_class_evidence_intake/submission kit spaced'"
|
|
assert quoted_spaced_dir in spaced_kit_payload["commands"]["validate_intake"], spaced_kit_payload["commands"]
|
|
assert quoted_spaced_dir in spaced_kit_payload["commands"]["refresh_ledger"], spaced_kit_payload["commands"]
|
|
spaced_readme = (spaced_kit_dir / "README.md").read_text(encoding="utf-8")
|
|
assert quoted_spaced_dir in spaced_readme, spaced_readme
|
|
|
|
existing_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout")
|
|
existing_payload = json.loads(existing_proc.stdout)
|
|
assert existing_payload["summary"]["existing_count"] == 1, existing_payload["summary"]
|
|
assert existing_payload["files"][0]["status"] == "exists", existing_payload["files"]
|
|
|
|
valid_dir = TMP / "valid_submissions"
|
|
valid_dir.mkdir()
|
|
(valid_dir / "provider-holdout.json").write_text(
|
|
json.dumps(provider_submission(valid=True), ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
valid_payload = run_intake("--submissions-dir", str(valid_dir))
|
|
assert valid_payload["ok"] is True, valid_payload
|
|
assert valid_payload["summary"]["decision"] == "source-evidence-incomplete", valid_payload["summary"]
|
|
assert valid_payload["summary"]["valid_submission_count"] == 1, valid_payload["summary"]
|
|
assert valid_payload["summary"]["valid_packet_source_incomplete_count"] == 1, valid_payload["summary"]
|
|
assert valid_payload["summary"]["operator_checklist_ready_count"] == 0, valid_payload["summary"]
|
|
assert valid_payload["summary"]["ready_for_ledger_review"] is False, valid_payload["summary"]
|
|
assert valid_payload["summary"]["ready_to_claim_world_class"] is False, valid_payload["summary"]
|
|
assert valid_payload["submissions"][0]["status"] == "pass", valid_payload["submissions"]
|
|
assert valid_payload["submissions"][0]["artifact_integrity"]["artifact_existing_count"] == 1, valid_payload["submissions"]
|
|
assert valid_payload["submissions"][0]["artifact_integrity"]["artifact_sha256_verified_count"] == 1, valid_payload["submissions"]
|
|
valid_checklist = {item["evidence_key"]: item for item in valid_payload["operator_checklist"]}
|
|
assert valid_checklist["provider-holdout"]["readiness"] == "source-evidence-incomplete", valid_checklist["provider-holdout"]
|
|
assert "source evidence checks still do not pass" in valid_checklist["provider-holdout"]["blocking_reason"], valid_checklist["provider-holdout"]
|
|
assert valid_checklist["provider-holdout"]["submission_path"].endswith("tests/tmp_world_class_evidence_intake/valid_submissions/provider-holdout.json"), valid_checklist["provider-holdout"]
|
|
assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["validate_intake"], valid_checklist["provider-holdout"]
|
|
assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["submission_review"], valid_checklist["provider-holdout"]
|
|
assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["refresh_ledger"], valid_checklist["provider-holdout"]
|
|
|
|
placeholder_dir = TMP / "placeholder_submissions"
|
|
placeholder_dir.mkdir()
|
|
placeholder_submission = provider_submission(valid=True)
|
|
placeholder_submission["submitted_by"] = "operator with provider credentials"
|
|
placeholder_submission["submitted_at"] = "YYYY-MM-DD"
|
|
(placeholder_dir / "provider-holdout.json").write_text(
|
|
json.dumps(placeholder_submission, ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
placeholder_payload = run_intake("--submissions-dir", str(placeholder_dir))
|
|
assert placeholder_payload["ok"] is False, placeholder_payload
|
|
assert placeholder_payload["summary"]["decision"] == "fix-intake", placeholder_payload["summary"]
|
|
assert placeholder_payload["summary"]["invalid_submission_count"] == 1, placeholder_payload["summary"]
|
|
placeholder_errors = placeholder_payload["submissions"][0]["errors"]
|
|
assert any("submitted_by must not use template placeholder text" in error for error in placeholder_errors), placeholder_errors
|
|
assert any("submitted_at must use YYYY-MM-DD" in error for error in placeholder_errors), placeholder_errors
|
|
|
|
invalid_dir = TMP / "invalid_submissions"
|
|
invalid_dir.mkdir()
|
|
(invalid_dir / "provider-holdout.json").write_text(
|
|
json.dumps(provider_submission(valid=False), ensure_ascii=False, indent=2) + "\n",
|
|
encoding="utf-8",
|
|
)
|
|
invalid_payload = run_intake("--submissions-dir", str(invalid_dir))
|
|
assert invalid_payload["ok"] is False, invalid_payload
|
|
assert invalid_payload["summary"]["decision"] == "fix-intake", invalid_payload["summary"]
|
|
assert invalid_payload["summary"]["invalid_submission_count"] == 1, invalid_payload["summary"]
|
|
assert invalid_payload["summary"]["operator_checklist_ready_count"] == 0, invalid_payload["summary"]
|
|
assert any("raw content" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"]
|
|
assert any("sha256" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"]
|
|
assert any("attestation.real_external_or_human_evidence" in error for error in invalid_payload["submissions"][0]["errors"]), invalid_payload["submissions"]
|
|
invalid_checklist = {item["evidence_key"]: item for item in invalid_payload["operator_checklist"]}
|
|
assert invalid_checklist["provider-holdout"]["readiness"] == "fix-submission", invalid_checklist["provider-holdout"]
|
|
|
|
unrelated_dir = TMP / "unrelated_artifact_submissions"
|
|
unrelated_dir.mkdir()
|
|
(unrelated_dir / "provider-holdout.json").write_text(
|
|
json.dumps(provider_submission(valid=True, artifact_path="reports/context_budget.json"), ensure_ascii=False, indent=2)
|
|
+ "\n",
|
|
encoding="utf-8",
|
|
)
|
|
unrelated_payload = run_intake("--submissions-dir", str(unrelated_dir))
|
|
assert unrelated_payload["ok"] is False, unrelated_payload
|
|
assert unrelated_payload["summary"]["decision"] == "fix-intake", unrelated_payload["summary"]
|
|
assert unrelated_payload["summary"]["invalid_submission_count"] == 1, unrelated_payload["summary"]
|
|
unrelated_errors = unrelated_payload["submissions"][0]["errors"]
|
|
assert any("required evidence artifact reports/output_execution_runs.json" in error for error in unrelated_errors), unrelated_errors
|
|
assert unrelated_payload["submissions"][0]["artifact_integrity"]["artifact_sha256_verified_count"] == 1, unrelated_payload["submissions"]
|
|
assert unrelated_payload["submissions"][0]["artifact_integrity"]["required_artifact_verified_count"] == 0, unrelated_payload["submissions"]
|
|
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|