frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Require reviewer approval for world-class evidence

Browse Source
This commit is contained in:
yaojingang
2026-06-16 22:32:44 +08:00
parent 1e1cb52f5d
commit a5bdd57ea1
23 changed files with 112 additions and 31 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evidence/world_class/intake.schema.json
+5 -1
View File
@@ -150,7 +150,8 @@
"real_external_or_human_evidence",
"reviewer_or_operator_identity_present",
"artifact_refs_reviewed",
"privacy_contract_satisfied"
"privacy_contract_satisfied",
"ledger_reviewer_approved"
],
"properties": {
"real_external_or_human_evidence": {
@@ -165,6 +166,9 @@
"privacy_contract_satisfied": {
"type": "boolean"
},
"ledger_reviewer_approved": {
"type": "boolean"
},
"notes": {
"type": "string"
}
evidence/world_class/templates/human-adjudication.intake.json
+1
View File
@@ -44,6 +44,7 @@
"reviewer_or_operator_identity_present": false,
"artifact_refs_reviewed": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"notes": "Copy this template into submissions/ after a real reviewer completes blind decisions."
}
}
evidence/world_class/templates/native-client-telemetry.intake.json
+1
View File
@@ -45,6 +45,7 @@
"reviewer_or_operator_identity_present": false,
"artifact_refs_reviewed": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"notes": "Copy this template into submissions/ after real client events have been imported."
}
}
evidence/world_class/templates/native-permission-enforcement.intake.json
+1
View File
@@ -53,6 +53,7 @@
"reviewer_or_operator_identity_present": false,
"artifact_refs_reviewed": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"notes": "Copy this template into submissions/ only after a real client or installer guard exists."
}
}
evidence/world_class/templates/provider-holdout.intake.json
+1
View File
@@ -42,6 +42,7 @@
"reviewer_or_operator_identity_present": false,
"artifact_refs_reviewed": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"notes": "Copy this template into submissions/ and set all booleans truthfully after the real provider run."
}
}
reports/architecture_maintainability.json
+2 -2
View File
@@ -31,7 +31,7 @@
},
{
"path": "scripts/world_class_evidence_contract.py",
"lines": 686,
"lines": 687,
"kind": "internal-module",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
@@ -52,7 +52,7 @@
},
{
"path": "tests/verify_world_class_evidence_intake.py",
"lines": 660,
"lines": 663,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
reports/architecture_maintainability.md
+2 -2
View File
@@ -34,10 +34,10 @@ No near-threshold files found.
| File | Lines | Kind | Severity |
| --- | ---: | --- | --- |
| `tests/verify_yao_cli.py` | `696` | `test` | `pass` |
| `scripts/world_class_evidence_contract.py` | `686` | `internal-module` | `pass` |
| `scripts/world_class_evidence_contract.py` | `687` | `internal-module` | `pass` |
| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` |
| `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` |
| `tests/verify_world_class_evidence_intake.py` | `660` | `test` | `pass` |
| `tests/verify_world_class_evidence_intake.py` | `663` | `test` | `pass` |
| `scripts/render_skill_os2_coverage.py` | `649` | `cli-script` | `pass` |
| `scripts/render_review_studio.py` | `647` | `cli-script` | `pass` |
| `scripts/render_reference_synthesis.py` | `644` | `cli-script` | `pass` |
reports/context_budget.json
+6 -6
View File
@@ -6,15 +6,15 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 797,
"other_text_tokens": 1055458,
"other_text_tokens": 1056256,
"estimated_initial_load_tokens": 990,
"estimated_total_text_tokens": 1056255,
"deferred_resource_tokens": 489507,
"estimated_total_text_tokens": 1057053,
"deferred_resource_tokens": 489692,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 429409,
"estimated_tokens": 429594,
"file_count": 135
},
{
@@ -36,7 +36,7 @@
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 429409,
"estimated_tokens": 429594,
"file_count": 135
}
],
@@ -59,7 +59,7 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 429409,
"estimated_tokens": 429594,
"file_count": 135,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
| root | `.` | `production` | 1000 | 990 | 797 | 489507 | `governed` | scripts:429409 | 131.3 | - | ok |
| root | `.` | `production` | 1000 | 990 | 797 | 489692 | `governed` | scripts:429594 | 131.3 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
reports/context_budget_summary.json
+3 -3
View File
@@ -8,11 +8,11 @@
"budget_limit": 1000,
"initial_tokens": 990,
"skill_body_tokens": 797,
"deferred_resource_tokens": 489507,
"deferred_resource_tokens": 489692,
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 429409,
"estimated_tokens": 429594,
"file_count": 135
}
],
@@ -35,7 +35,7 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 429409,
"estimated_tokens": 429594,
"file_count": 135,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 226,
"package_sha256": "c65410d8e0bb1ea55fc52ea0b2c557f9590b232722b51e77e059595f64bf668f"
"package_sha256": "95b883a7258f8b201a7f94b47a35fa3f665ac65160def6d41764f20e2cdb5e4d"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `226`
- Package SHA256: `c65410d8e0bb1ea55fc52ea0b2c557f9590b232722b51e77e059595f64bf668f`
- Package SHA256: `95b883a7258f8b201a7f94b47a35fa3f665ac65160def6d41764f20e2cdb5e4d`
## Failures
reports/world_class_evidence_ledger.json
+7
View File
@@ -12,6 +12,7 @@
"human_pending_count": 1,
"external_pending_count": 3,
"submitted_entry_count": 0,
"reviewer_approved_submission_count": 0,
"missing_submission_count": 4,
"invalid_submission_count": 0,
"source_check_count": 13,
@@ -28,6 +29,7 @@
"human_pending_count": 1,
"external_pending_count": 3,
"submitted_entry_count": 0,
"reviewer_approved_submission_count": 0,
"missing_submission_count": 4,
"invalid_submission_count": 0,
"source_check_count": 13,
@@ -53,6 +55,7 @@
"human_pending_count",
"external_pending_count",
"submitted_entry_count",
"reviewer_approved_submission_count",
"missing_submission_count",
"invalid_submission_count",
"source_check_count",
@@ -154,6 +157,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
@@ -272,6 +276,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
@@ -380,6 +385,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
@@ -481,6 +487,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
reports/world_class_evidence_ledger.md
+5 -4
View File
@@ -15,6 +15,7 @@ Generated at: `2026-06-16`
- human pending: `1`
- external pending: `3`
- submitted entries: `0`
- reviewer approved submissions: `0`
- submitted but pending: `0`
- source accepted without valid submission: `0`
- invalid submissions: `0`
@@ -37,7 +38,7 @@ This ledger records the current evidence state. It requires both passing source
- source status: `external_required`
- observed state: `{"model_executed_count": 0, "timing_observed_count": 10, "token_observed_count": 0, "accepted": false}`
- source checks: `1` pass / `3` total
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_counts_as_completion": false}`
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_counts_as_completion": false}`
### Provenance Requirements
@@ -80,7 +81,7 @@ This ledger records the current evidence state. It requires both passing source
- source status: `human_required`
- observed state: `{"pair_count": 5, "judgment_count": 0, "pending_count": 5, "invalid_decision_count": 0, "answer_revealed_count": 0, "accepted": false}`
- source checks: `2` pass / `4` total
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/human-adjudication.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_counts_as_completion": false}`
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/human-adjudication.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_counts_as_completion": false}`
### Provenance Requirements
@@ -128,7 +129,7 @@ This ledger records the current evidence state. It requires both passing source
- source status: `external_required`
- observed state: `{"native_enforcement_count": 0, "metadata_fallback_count": 4, "installer_enforcement_pass_count": 4, "installer_permission_failure_count": 0, "installer_enforcement_ready": true, "residual_risk_count": 4, "failure_count": 0, "accepted": false}`
- source checks: `2` pass / `3` total
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-permission-enforcement.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_counts_as_completion": false}`
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-permission-enforcement.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_counts_as_completion": false}`
### Provenance Requirements
@@ -173,7 +174,7 @@ This ledger records the current evidence state. It requires both passing source
- source status: `external_required`
- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}`
- source checks: `1` pass / `3` total
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_counts_as_completion": false}`
- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_counts_as_completion": false}`
### Provenance Requirements
reports/world_class_operator_runbook.json
+4
View File
@@ -127,6 +127,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
@@ -259,6 +260,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
@@ -380,6 +382,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
@@ -495,6 +498,7 @@
"artifact_ref_count": 0,
"attested_real_evidence": false,
"privacy_contract_satisfied": false,
"ledger_reviewer_approved": false,
"ledger_counts_as_completion": false
},
"anti_overclaim": {
scripts/prepare_world_class_submission_kit.py
+3 -3
View File
@@ -335,7 +335,7 @@ def build_handoff_steps(
"label": "Edit submission packet",
"status": "manual",
"command": "",
"completion_signal": "template_only is false only after real evidence exists and all attestation fields are truthful.",
"completion_signal": "template_only is false only after real evidence exists; ledger_reviewer_approved remains false until reviewer approval.",
"counts_as_completion": False,
"blocking_condition": "Raw prompts, outputs, transcripts, credentials, or private content must not be included.",
},
@@ -344,7 +344,7 @@ def build_handoff_steps(
"label": "Validate intake contract",
"status": "pending",
"command": commands["validate_intake"],
"completion_signal": "world_class_evidence_intake reports valid submissions and no invalid packets.",
"completion_signal": "world_class_evidence_intake reports valid submissions and no invalid packets before reviewer approval.",
"counts_as_completion": False,
"blocking_condition": "A valid packet is ready for ledger review but is not accepted evidence by itself.",
},
@@ -353,7 +353,7 @@ def build_handoff_steps(
"label": "Review submission queue",
"status": "pending",
"command": commands["review_submission"],
"completion_signal": "world_class_submission_review shows ready-for-ledger-review for the submitted key.",
"completion_signal": "world_class_submission_review shows ready-for-ledger-review before ledger_reviewer_approved is set.",
"counts_as_completion": False,
"blocking_condition": "Reviewer queue output is advisory and cannot accept evidence.",
},
scripts/render_world_class_evidence_ledger.py
+13 -5
View File
@@ -102,6 +102,7 @@ TOP_LEVEL_SUMMARY_FIELDS = [
"human_pending_count",
"external_pending_count",
"submitted_entry_count",
"reviewer_approved_submission_count",
"missing_submission_count",
"invalid_submission_count",
"source_check_count",
@@ -125,10 +126,11 @@ def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Pat
return {
"status": load_status,
"path": rel_path(path, skill_dir),
"artifact_ref_count": 0,
"attested_real_evidence": False,
"privacy_contract_satisfied": False,
"ledger_counts_as_completion": False,
"artifact_ref_count": 0,
"attested_real_evidence": False,
"privacy_contract_satisfied": False,
"ledger_reviewer_approved": False,
"ledger_counts_as_completion": False,
}
validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False)
refs = payload.get("artifact_refs", [])
@@ -146,6 +148,7 @@ def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Pat
"attested_real_evidence": attestation.get("real_external_or_human_evidence") is True,
"reviewer_or_operator_identity_present": attestation.get("reviewer_or_operator_identity_present") is True,
"privacy_contract_satisfied": attestation.get("privacy_contract_satisfied") is True,
"ledger_reviewer_approved": attestation.get("ledger_reviewer_approved") is True,
"errors": validation.get("errors", []),
"ledger_counts_as_completion": False,
}
@@ -157,7 +160,7 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) ->
source_checklist = build_source_checklist([{"key": task["key"], "observed_state": state}])
source_summary = summarize_source_checklist(source_checklist)
source_accepted = bool(source_checklist) and source_summary["source_blocked_count"] == 0
accepted = source_accepted and submission.get("status") == "submitted"
accepted = source_accepted and submission.get("status") == "submitted" and submission.get("ledger_reviewer_approved") is True
return {
"key": task["key"],
"label": task["label"],
@@ -200,6 +203,9 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
human_pending_count = sum(1 for entry in entries if entry["category"] == "human" and entry["status"] == "pending")
external_pending_count = sum(1 for entry in entries if entry["category"] == "external" and entry["status"] == "pending")
submitted_entry_count = sum(1 for entry in entries if entry["submission_state"]["status"] == "submitted")
reviewer_approved_submission_count = sum(
1 for entry in entries if entry["submission_state"].get("ledger_reviewer_approved") is True
)
missing_submission_count = sum(1 for entry in entries if entry["submission_state"]["status"] == "missing")
invalid_submission_count = sum(1 for entry in entries if entry["submission_state"]["status"] in {"invalid-json", "invalid-contract"})
submitted_but_pending_count = sum(
@@ -219,6 +225,7 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non
"human_pending_count": human_pending_count,
"external_pending_count": external_pending_count,
"submitted_entry_count": submitted_entry_count,
"reviewer_approved_submission_count": reviewer_approved_submission_count,
"missing_submission_count": missing_submission_count,
"invalid_submission_count": invalid_submission_count,
**source_summary,
@@ -282,6 +289,7 @@ def render_markdown(ledger: dict[str, Any]) -> str:
f"- human pending: `{summary['human_pending_count']}`",
f"- external pending: `{summary['external_pending_count']}`",
f"- submitted entries: `{summary['submitted_entry_count']}`",
f"- reviewer approved submissions: `{summary.get('reviewer_approved_submission_count', 0)}`",
f"- submitted but pending: `{summary['submitted_but_pending_count']}`",
f"- source accepted without valid submission: `{summary.get('source_accepted_without_valid_submission_count', 0)}`",
f"- invalid submissions: `{summary['invalid_submission_count']}`",
scripts/world_class_evidence_contract.py
+1
View File
@@ -41,6 +41,7 @@ REQUIRED_ATTESTATION_TRUE = [
"reviewer_or_operator_identity_present",
"artifact_refs_reviewed",
"privacy_contract_satisfied",
"ledger_reviewer_approved",
]
EXPECTED_SOURCE_TYPES = {
"provider-holdout": "provider-output-eval",
scripts/world_class_submission_kit_rendering.py
+2 -2
View File
@@ -25,7 +25,7 @@ def render_readme(report: dict[str, Any]) -> str:
"2. Edit the matching JSON draft with only aggregate artifact references and provenance metadata.",
"3. Set `template_only` to `false` only after real evidence exists.",
"4. Set attestation booleans truthfully; do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content.",
"5. Validate the packet before asking the ledger reviewer to accept it.",
"5. Validate the packet before asking the ledger reviewer to set `attestation.ledger_reviewer_approved` truthfully.",
"6. Optional artifact prefill only inserts SHA-256 digests for current local aggregate artifacts; it does not mark a draft as real evidence.",
"",
"## Commands",
@@ -438,7 +438,7 @@ def render_html(report: dict[str, Any]) -> str:
<div class="stats">{stat_html}</div>
</section>
<section class="section two-col" id="workflow">
<article class="panel"><h2>Workflow</h2><ol><li>Run the real provider, human review, native permission, or native client telemetry work first.</li><li>Edit the matching JSON draft with aggregate artifact references and provenance metadata.</li><li>Set template_only to false only after real evidence exists.</li><li>Use prefilled SHA-256 values as convenience data, not evidence acceptance.</li><li>Validate intake, refresh the ledger, then guard public claims.</li></ol></article>
<article class="panel"><h2>Workflow</h2><ol><li>Run the real provider, human review, native permission, or native client telemetry work first.</li><li>Edit the matching JSON draft with aggregate artifact references and provenance metadata.</li><li>Set template_only to false only after real evidence exists.</li><li>Use prefilled SHA-256 values as convenience data, not evidence acceptance.</li><li>Validate intake, ask the ledger reviewer to approve explicitly, refresh the ledger, then guard public claims.</li></ol></article>
<aside class="panel"><h2>Commands</h2><ul class="commands">{render_html_commands(report['commands'])}</ul></aside>
</section>
<section class="section" id="handoff"><h2>Operator Handoff</h2><p class="muted">These ordered steps make the operator-to-reviewer handoff auditable. They are procedural guardrails and never count as world-class evidence.</p><div class="handoff-grid">{handoff_html}</div></section>
tests/verify_world_class_evidence_intake.py
+3
View File
@@ -86,6 +86,7 @@ def provider_submission(*, valid: bool, artifact_path: str = "reports/output_exe
"reviewer_or_operator_identity_present": valid,
"artifact_refs_reviewed": valid,
"privacy_contract_satisfied": valid,
"ledger_reviewer_approved": valid,
},
}
@@ -136,6 +137,7 @@ def external_submission(
"reviewer_or_operator_identity_present": True,
"artifact_refs_reviewed": True,
"privacy_contract_satisfied": True,
"ledger_reviewer_approved": True,
},
}
@@ -369,6 +371,7 @@ def human_submission(skill_root: Path, *, reviewer: str = "Yao QA") -> dict:
"reviewer_or_operator_identity_present": True,
"artifact_refs_reviewed": True,
"privacy_contract_satisfied": True,
"ledger_reviewer_approved": True,
},
}
tests/verify_world_class_evidence_ledger.py
+47
View File
@@ -60,6 +60,7 @@ def provider_submission(artifact_root: Path = ROOT, artifact_path: str = "report
"reviewer_or_operator_identity_present": True,
"artifact_refs_reviewed": True,
"privacy_contract_satisfied": True,
"ledger_reviewer_approved": True,
},
}
@@ -340,6 +341,51 @@ def main() -> None:
for error in placeholder_accepted_provider["submission_state"]["errors"]
), placeholder_accepted_provider
unapproved_submissions = TMP / "unapproved_submissions"
unapproved_submissions.mkdir()
unapproved_submission = provider_submission(accepted_source_skill)
unapproved_submission["attestation"]["ledger_reviewer_approved"] = False
(unapproved_submissions / "provider-holdout.json").write_text(
json.dumps(unapproved_submission, ensure_ascii=False, indent=2) + "\n",
encoding="utf-8",
)
unapproved_proc = subprocess.run(
[
sys.executable,
str(SCRIPT),
str(accepted_source_skill),
"--output-json",
str(TMP / "unapproved_provider_ledger.json"),
"--output-md",
str(TMP / "unapproved_provider_ledger.md"),
"--submissions-dir",
str(unapproved_submissions),
"--generated-at",
"2026-06-13",
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
unapproved_payload = json.loads(unapproved_proc.stdout)
unapproved_summary = unapproved_payload["summary"]
assert unapproved_summary["source_accepted_count"] == 1, unapproved_summary
assert unapproved_summary["submitted_entry_count"] == 0, unapproved_summary
assert unapproved_summary["reviewer_approved_submission_count"] == 0, unapproved_summary
assert unapproved_summary["accepted_count"] == 0, unapproved_summary
unapproved_provider = {
entry["key"]: entry for entry in unapproved_payload["entries"]
}["provider-holdout"]
assert unapproved_provider["source_accepted"] is True, unapproved_provider
assert unapproved_provider["status"] == "pending", unapproved_provider
assert unapproved_provider["submission_state"]["status"] == "invalid-contract", unapproved_provider
assert unapproved_provider["submission_state"]["ledger_reviewer_approved"] is False, unapproved_provider
assert any(
"attestation.ledger_reviewer_approved must be true" in error
for error in unapproved_provider["submission_state"]["errors"]
), unapproved_provider
accepted_submissions = TMP / "accepted_submissions"
accepted_submissions.mkdir()
(accepted_submissions / "provider-holdout.json").write_text(
@@ -369,6 +415,7 @@ def main() -> None:
accepted_summary = accepted_payload["summary"]
assert accepted_summary["source_accepted_count"] == 1, accepted_summary
assert accepted_summary["accepted_count"] == 1, accepted_summary
assert accepted_summary["reviewer_approved_submission_count"] == 1, accepted_summary
assert accepted_summary["pending_count"] == 3, accepted_summary
assert accepted_summary["source_accepted_without_valid_submission_count"] == 0, accepted_summary
accepted_provider = {entry["key"]: entry for entry in accepted_payload["entries"]}["provider-holdout"]
tests/verify_world_class_operator_runbook.py
+1
View File
@@ -58,6 +58,7 @@ def provider_submission() -> dict:
"reviewer_or_operator_identity_present": True,
"artifact_refs_reviewed": True,
"privacy_contract_satisfied": True,
"ledger_reviewer_approved": True,
},
}
tests/verify_world_class_submission_review.py
+1
View File
@@ -60,6 +60,7 @@ def provider_submission(*, valid: bool = True) -> dict:
"reviewer_or_operator_identity_present": valid,
"artifact_refs_reviewed": valid,
"privacy_contract_satisfied": valid,
"ledger_reviewer_approved": valid,
},
}