diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index 64261a5..0857a20 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -197,6 +197,68 @@ def build_artifact_checklist(skill_dir: Path, items: list[dict[str, Any]]) -> li return rows +SOURCE_CHECK_SPECS = { + "provider-holdout": [ + ("Provider model run", "model_executed_count", ">0", "Run provider-backed output-exec with real credentials."), + ("Timing observed", "timing_observed_count", ">0", "Provider execution should record timing metadata."), + ("Token usage observed", "token_observed_count", ">0", "Provider execution should return non-estimated token usage."), + ], + "human-adjudication": [ + ("Review pairs exist", "pair_count", ">0", "Generate the blind A/B review pack."), + ("No pending decisions", "pending_count", "==0", "Record a reviewer choice for every pair."), + ("Judgments complete", "judgment_count", "==pair_count", "Every pair needs one valid human judgment."), + ("No invalid decisions", "invalid_decision_count", "==0", "Fix malformed winner/confidence entries."), + ], + "native-permission-enforcement": [ + ("Native enforcement", "native_enforcement_count", ">0", "Collect real target-client or external runtime guard proof."), + ("Probe failures", "failure_count", "==0", "Runtime permission probes must stay clean."), + ("Installer support", "installer_enforcement_ready", "true", "Installer enforcement is supporting evidence, not native proof."), + ], + "native-client-telemetry": [ + ("External events", "external_source_events", ">0", "Import at least one metadata-only event from a real client."), + ("Adoption sample", "adoption_sample_count", ">0", "Telemetry must include adoption outcome evidence."), + ("Raw content blocked", "raw_content_allowed", "false", "Telemetry must stay metadata-only."), + ], +} + + +def source_check_passed(actual: Any, expected: str, observed_state: dict[str, Any]) -> bool: + if expected == ">0": + return isinstance(actual, (int, float)) and actual > 0 + if expected == "==0": + return actual == 0 + if expected == "true": + return actual is True + if expected == "false": + return actual is False + if expected == "==pair_count": + return actual == observed_state.get("pair_count") and isinstance(actual, (int, float)) and actual > 0 + return False + + +def build_source_checklist(items: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for item in items: + key = str(item.get("evidence_key", "")) + observed_state = item.get("observed_state", {}) if isinstance(item.get("observed_state", {}), dict) else {} + for label, field, expected, next_action in SOURCE_CHECK_SPECS.get(key, []): + actual = observed_state.get(field) + passed = source_check_passed(actual, expected, observed_state) + rows.append( + { + "evidence_key": key, + "label": label, + "field": field, + "expected": expected, + "actual": actual, + "status": "pass" if passed else "blocked", + "source_accepted": item.get("source_accepted") is True, + "next_action": next_action, + } + ) + return rows + + def render_readme(report: dict[str, Any]) -> str: commands = report["commands"] lines = [ @@ -243,6 +305,21 @@ def render_readme(report: dict[str, Any]) -> str: lines.append( f"| `{item['evidence_key']}` | `{item['path']}` | `{item['status']}` | `{digest}` |" ) + lines.extend( + [ + "", + "## Source Evidence Snapshot", + "", + "These checks explain why a draft is not ready for ledger acceptance yet. They mirror current aggregate reports and do not accept evidence by themselves.", + "", + "| Evidence | Check | Current | Expected | Status |", + "| --- | --- | --- | --- | --- |", + ] + ) + for item in report.get("source_checklist", []): + lines.append( + f"| `{item['evidence_key']}` | {item['label']} | `{item['actual']}` | `{item['expected']}` | `{item['status']}` |" + ) lines.extend( [ "", @@ -329,6 +406,36 @@ def render_html_artifact_checklist(items: list[dict[str, Any]]) -> str: ) +def render_html_source_checklist(items: list[dict[str, Any]]) -> str: + if not items: + return "

No source checks were listed for the requested evidence.

" + return "".join( + """ +
+
+ {key} +

{label}

+
+
+
Field
{field}
+
Current
{actual}
+
Expected
{expected}
+
Action
{action}
+
+
+ """.format( + status=html_text(item.get("status", "")), + key=html_text(item.get("evidence_key", "")), + label=html_text(item.get("label", "")), + field=html_text(item.get("field", "")), + actual=html_text(item.get("actual", "")), + expected=html_text(item.get("expected", "")), + action=html_text(item.get("next_action", "")), + ) + for item in items + ) + + def render_html_item(item: dict[str, Any]) -> str: must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} return f""" @@ -375,6 +482,7 @@ def render_html(report: dict[str, Any]) -> str: stat_html = "".join(f"
{html_text(label)}{html_text(value)}
" for label, value in stats) evidence_html = "".join(render_html_item(item) for item in report.get("evidence_items", [])) artifact_html = render_html_artifact_checklist(report.get("artifact_checklist", [])) + source_html = render_html_source_checklist(report.get("source_checklist", [])) return f""" @@ -407,12 +515,14 @@ def render_html(report: dict[str, Any]) -> str: .section {{ padding:32px 0; border-bottom:1px solid var(--line); }} .panel {{ padding:20px; }} .two-col {{ display:grid; grid-template-columns:minmax(0, .45fr) minmax(0, 1fr); gap:18px; align-items:start; }} - .draft-grid, .evidence-grid, .artifact-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} - .draft-card, .evidence-card, .artifact-card {{ padding:18px; min-width:0; }} + .draft-grid, .evidence-grid, .artifact-grid, .source-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} + .draft-card, .evidence-card, .artifact-card, .source-card {{ padding:18px; min-width:0; }} .draft-card.written, .draft-card.exists {{ border-left:4px solid var(--pass); }} .draft-card.skipped {{ border-left:4px solid var(--warn); }} .evidence-card.awaiting-submission, .evidence-card.fix-submission, .evidence-card.fix-template, .artifact-card.missing, .artifact-card.glob-no-match, .artifact-card.unsafe-path, .artifact-card.raw-content-disallowed {{ border-left:4px solid var(--warn); }} .artifact-card.ready {{ border-left:4px solid var(--pass); }} + .source-card.blocked {{ border-left:4px solid var(--warn); }} + .source-card.pass {{ border-left:4px solid var(--pass); }} dl {{ display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }} dt {{ color:var(--ink); }} dd {{ margin:0; min-width:0; overflow-wrap:anywhere; }} @@ -426,11 +536,11 @@ def render_html(report: dict[str, Any]) -> str: .mini-grid li, .notice li {{ overflow-wrap:anywhere; }} .notice {{ background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; }} .errors {{ color:var(--warn); }} - @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} + @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} - +
Evidence Intake @@ -444,6 +554,7 @@ def render_html(report: dict[str, Any]) -> str:

Drafts

{render_html_files(report['files'])}

Artifact Checklist

Copy concrete paths and SHA-256 digests from here into artifact_refs after real evidence exists. Glob patterns are expanded for operator convenience only.

{artifact_html}
+

Source Evidence Snapshot

This section shows current aggregate source checks. It explains remaining blockers without changing the ledger.

{source_html}

Evidence Requirements

{evidence_html}

Safety Boundary

  • Drafts never count as accepted ledger evidence.
  • Valid intake means ready for ledger review, not world-class completion.
  • Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content.
@@ -467,6 +578,7 @@ def build_submission_kit( template_results = template_result_by_key(intake) files = [copy_template(skill_dir, output_dir, item, template_results, overwrite) for item in items] artifact_checklist = build_artifact_checklist(skill_dir, items) + source_checklist = build_source_checklist(items) manifest_path = output_dir / "submission_manifest.json" readme_path = output_dir / "README.md" output_html = output_html or (output_dir / "index.html") @@ -476,6 +588,8 @@ def build_submission_kit( artifact_ready_count = sum(1 for item in artifact_checklist if item.get("artifact_ref_ready")) artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready")) artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required")) + source_pass_count = sum(1 for item in source_checklist if item.get("status") == "pass") + source_blocked_count = sum(1 for item in source_checklist if item.get("status") != "pass") ok = not unknown_keys and skipped_count == 0 report = { "schema_version": "1.0", @@ -494,6 +608,9 @@ def build_submission_kit( "artifact_ready_count": artifact_ready_count, "artifact_missing_count": artifact_missing_count, "artifact_glob_expansion_count": artifact_glob_count, + "source_check_count": len(source_checklist), + "source_pass_count": source_pass_count, + "source_blocked_count": source_blocked_count, "drafts_count_as_evidence": False, "ledger_counts_submission_as_completion": False, "decision": "submission-kit-ready" if ok else "fix-submission-kit", @@ -501,6 +618,7 @@ def build_submission_kit( "unknown_evidence_keys": unknown_keys, "files": files, "artifact_checklist": artifact_checklist, + "source_checklist": source_checklist, "evidence_items": items, "commands": { "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {shell_path(output_dir, skill_dir)}", diff --git a/scripts/render_world_class_evidence_intake.py b/scripts/render_world_class_evidence_intake.py index f07d3fd..9cbb75a 100644 --- a/scripts/render_world_class_evidence_intake.py +++ b/scripts/render_world_class_evidence_intake.py @@ -93,6 +93,8 @@ def build_operator_checklist( "blocking_reason": blocking_reason, "template_status": template_result.get("status", "missing") if template_result else "missing", "submission_status": submission_result.get("status", "missing") if submission_result else "missing", + "source_accepted": entry.get("source_accepted") is True, + "observed_state": entry.get("observed_state", {}), "template_path": rel_path(template_path, skill_dir), "submission_path": rel_path(submission_path, skill_dir), "commands": { diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index a37d3ac..4272806 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -167,6 +167,9 @@ def main() -> None: assert "provider-backed model run" in checklist["provider-holdout"]["must_collect"]["provenance_requirements"], checklist["provider-holdout"] assert "reports/output_execution_runs.json summary.model_executed_count > 0" in checklist["provider-holdout"]["must_collect"]["success_checks"], checklist["provider-holdout"] assert checklist["provider-holdout"]["anti_overclaim"]["local_command_runner_counts_as_provider_model"] is False, checklist["provider-holdout"] + assert checklist["provider-holdout"]["source_accepted"] is False, checklist["provider-holdout"] + assert checklist["provider-holdout"]["observed_state"]["model_executed_count"] == 0, checklist["provider-holdout"] + assert checklist["provider-holdout"]["observed_state"]["timing_observed_count"] > 0, checklist["provider-holdout"] markdown = default_md.read_text(encoding="utf-8") assert "World-Class Evidence Intake" in markdown, markdown assert "ready to claim world-class: `false`" in markdown, markdown @@ -191,6 +194,9 @@ def main() -> None: assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"] assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"] assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"] + assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"] assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"] @@ -200,23 +206,32 @@ def main() -> None: assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows + source_rows = {item["field"]: item for item in kit_payload["source_checklist"]} + assert source_rows["model_executed_count"]["status"] == "blocked", source_rows + assert source_rows["model_executed_count"]["actual"] == 0, source_rows + assert source_rows["timing_observed_count"]["status"] == "pass", source_rows + assert source_rows["token_observed_count"]["status"] == "blocked", source_rows kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) assert kit_draft["template_only"] is True, kit_draft assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8")) assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"] assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"] + assert kit_manifest["source_checklist"] == kit_payload["source_checklist"], kit_manifest["source_checklist"] assert kit_manifest["artifacts"]["html"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/index.html"), kit_manifest["artifacts"] kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8") assert "Drafts are not accepted evidence." in kit_readme, kit_readme assert "validate intake" in kit_readme, kit_readme assert "Artifact Checklist" in kit_readme, kit_readme + assert "Source Evidence Snapshot" in kit_readme, kit_readme assert "reports/output_execution_runs.json" in kit_readme, kit_readme + assert "Provider model run" in kit_readme, kit_readme kit_html = (kit_dir / "index.html").read_text(encoding="utf-8") assert "World-Class Evidence Submission Kit" in kit_html, kit_html assert "Drafts are not accepted evidence" in kit_html, kit_html assert "provider-holdout" in kit_html, kit_html assert "Artifact Checklist" in kit_html, kit_html + assert "Source Evidence Snapshot" in kit_html, kit_html assert "World-Class Evidence Submission Kit" in kit_html, kit_html assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html