feat: reject placeholder evidence submissions

This commit is contained in:
yaojingang
2026-06-14 10:43:19 +08:00
parent 3f0b262e5f
commit 985eaee14b
21 changed files with 211 additions and 81 deletions
@@ -159,6 +159,7 @@ def main() -> None:
assert "`python3 scripts/yao.py world-class-ledger .`" in markdown, markdown
assert "Templates and planned work do not count as accepted evidence." in markdown, markdown
assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown
assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown
kit_dir = TMP / "submission_kit"
kit_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout")
@@ -219,6 +220,23 @@ def main() -> None:
assert valid_checklist["provider-holdout"]["submission_path"].endswith("tests/tmp_world_class_evidence_intake/valid_submissions/provider-holdout.json"), valid_checklist["provider-holdout"]
assert "tests/tmp_world_class_evidence_intake/valid_submissions" in valid_checklist["provider-holdout"]["commands"]["validate_intake"], valid_checklist["provider-holdout"]
placeholder_dir = TMP / "placeholder_submissions"
placeholder_dir.mkdir()
placeholder_submission = provider_submission(valid=True)
placeholder_submission["submitted_by"] = "operator with provider credentials"
placeholder_submission["submitted_at"] = "YYYY-MM-DD"
(placeholder_dir / "provider-holdout.json").write_text(
json.dumps(placeholder_submission, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
placeholder_payload = run_intake("--submissions-dir", str(placeholder_dir))
assert placeholder_payload["ok"] is False, placeholder_payload
assert placeholder_payload["summary"]["decision"] == "fix-intake", placeholder_payload["summary"]
assert placeholder_payload["summary"]["invalid_submission_count"] == 1, placeholder_payload["summary"]
placeholder_errors = placeholder_payload["submissions"][0]["errors"]
assert any("submitted_by must not use template placeholder text" in error for error in placeholder_errors), placeholder_errors
assert any("submitted_at must use YYYY-MM-DD" in error for error in placeholder_errors), placeholder_errors
invalid_dir = TMP / "invalid_submissions"
invalid_dir.mkdir()
(invalid_dir / "provider-holdout.json").write_text(
@@ -263,6 +263,50 @@ def main() -> None:
for error in unrelated_accepted_provider["submission_state"]["errors"]
), unrelated_accepted_provider
placeholder_accepted_submissions = TMP / "placeholder_accepted_submissions"
placeholder_accepted_submissions.mkdir()
placeholder_submission = provider_submission(accepted_source_skill)
placeholder_submission["submitted_by"] = "operator with provider credentials"
placeholder_submission["submitted_at"] = "YYYY-MM-DD"
(placeholder_accepted_submissions / "provider-holdout.json").write_text(
json.dumps(placeholder_submission, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
placeholder_accepted_proc = subprocess.run(
[
sys.executable,
str(SCRIPT),
str(accepted_source_skill),
"--output-json",
str(TMP / "placeholder_accepted_provider_ledger.json"),
"--output-md",
str(TMP / "placeholder_accepted_provider_ledger.md"),
"--submissions-dir",
str(placeholder_accepted_submissions),
"--generated-at",
"2026-06-13",
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
placeholder_accepted_payload = json.loads(placeholder_accepted_proc.stdout)
placeholder_accepted_summary = placeholder_accepted_payload["summary"]
assert placeholder_accepted_summary["source_accepted_count"] == 1, placeholder_accepted_summary
assert placeholder_accepted_summary["accepted_count"] == 0, placeholder_accepted_summary
assert placeholder_accepted_summary["invalid_submission_count"] == 1, placeholder_accepted_summary
placeholder_accepted_provider = {
entry["key"]: entry for entry in placeholder_accepted_payload["entries"]
}["provider-holdout"]
assert placeholder_accepted_provider["source_accepted"] is True, placeholder_accepted_provider
assert placeholder_accepted_provider["status"] == "pending", placeholder_accepted_provider
assert placeholder_accepted_provider["submission_state"]["status"] == "invalid-contract", placeholder_accepted_provider
assert any(
"submitted_by must not use template placeholder text" in error
for error in placeholder_accepted_provider["submission_state"]["errors"]
), placeholder_accepted_provider
accepted_submissions = TMP / "accepted_submissions"
accepted_submissions.mkdir()
(accepted_submissions / "provider-holdout.json").write_text(