diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index 476c858..64261a5 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -1,5 +1,6 @@ #!/usr/bin/env python3 import argparse +import hashlib import html import json import shlex @@ -9,6 +10,7 @@ from pathlib import Path from typing import Any from render_world_class_evidence_intake import build_intake +from world_class_evidence_contract import DISALLOWED_REAL_ARTIFACTS ROOT = Path(__file__).resolve().parent.parent @@ -36,6 +38,18 @@ def html_text(value: Any) -> str: return html.escape(str(value or ""), quote=True) +def has_glob_pattern(value: str) -> bool: + return any(token in value for token in ("*", "?", "[")) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + def requested_checklist_items(intake: dict[str, Any], evidence_keys: list[str]) -> list[dict[str, Any]]: items = intake.get("operator_checklist", []) if not evidence_keys: @@ -95,6 +109,94 @@ def copy_template( } +def artifact_row(skill_dir: Path, evidence_key: str, pattern: str, path: Path, status: str) -> dict[str, Any]: + exists = path.exists() + is_file = exists and path.is_file() + relative = rel_path(path, skill_dir) if exists else pattern + contains_raw_content = relative in DISALLOWED_REAL_ARTIFACTS + digest = sha256_file(path) if is_file and not contains_raw_content else "" + row_status = status if status else ("ready" if is_file else "missing") + if contains_raw_content: + row_status = "raw-content-disallowed" + return { + "evidence_key": evidence_key, + "source_pattern": pattern, + "path": relative, + "status": row_status, + "exists": exists, + "is_file": is_file, + "sha256": digest, + "artifact_ref_ready": bool(is_file and digest and not contains_raw_content), + "copy_path": relative if is_file else "", + "copy_sha256": digest, + "contains_raw_content": contains_raw_content, + "concrete_reference_required": has_glob_pattern(pattern), + } + + +def artifact_checklist_for_item(skill_dir: Path, item: dict[str, Any]) -> list[dict[str, Any]]: + key = str(item.get("evidence_key", "")) + must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} + artifacts = must_collect.get("evidence_artifacts", []) + if not isinstance(artifacts, list): + return [] + rows: list[dict[str, Any]] = [] + for artifact in artifacts: + pattern = str(artifact or "").strip() + if not pattern: + continue + if Path(pattern).is_absolute() or ".." in Path(pattern).parts: + rows.append( + { + "evidence_key": key, + "source_pattern": pattern, + "path": pattern, + "status": "unsafe-path", + "exists": False, + "is_file": False, + "sha256": "", + "artifact_ref_ready": False, + "copy_path": "", + "copy_sha256": "", + "contains_raw_content": False, + "concrete_reference_required": True, + } + ) + continue + if has_glob_pattern(pattern): + matches = sorted(path for path in skill_dir.glob(pattern) if path.is_file()) + if not matches: + rows.append( + { + "evidence_key": key, + "source_pattern": pattern, + "path": pattern, + "status": "glob-no-match", + "exists": False, + "is_file": False, + "sha256": "", + "artifact_ref_ready": False, + "copy_path": "", + "copy_sha256": "", + "contains_raw_content": False, + "concrete_reference_required": True, + } + ) + continue + for match in matches: + rows.append(artifact_row(skill_dir, key, pattern, match, "ready")) + continue + rows.append(artifact_row(skill_dir, key, pattern, skill_dir / pattern, "")) + return rows + + +def build_artifact_checklist(skill_dir: Path, items: list[dict[str, Any]]) -> list[dict[str, Any]]: + rows: list[dict[str, Any]] = [] + for item in items: + rows.extend(artifact_checklist_for_item(skill_dir, item)) + return rows + + def render_readme(report: dict[str, Any]) -> str: commands = report["commands"] lines = [ @@ -125,6 +227,22 @@ def render_readme(report: dict[str, Any]) -> str: ] for item in report["files"]: lines.append(f"| `{item['evidence_key']}` | `{item['output_path']}` | `{item['status']}` |") + lines.extend( + [ + "", + "## Artifact Checklist", + "", + "Use these paths and SHA-256 digests when filling `artifact_refs`. Glob patterns are expanded into concrete files; submissions must reference concrete paths, not globs.", + "", + "| Evidence | Path | Status | SHA-256 |", + "| --- | --- | --- | --- |", + ] + ) + for item in report.get("artifact_checklist", []): + digest = item.get("sha256") or "n/a" + lines.append( + f"| `{item['evidence_key']}` | `{item['path']}` | `{item['status']}` | `{digest}` |" + ) lines.extend( [ "", @@ -184,6 +302,33 @@ def render_html_files(files: list[dict[str, Any]]) -> str: ) +def render_html_artifact_checklist(items: list[dict[str, Any]]) -> str: + if not items: + return "

No required artifacts were listed for the requested evidence.

" + return "".join( + """ +
+
+ {key} +

{path}

+
+
+
Pattern
{pattern}
+
Status
{status}
+
SHA-256
{sha}
+
+
+ """.format( + status=html_text(item.get("status", "")), + key=html_text(item.get("evidence_key", "")), + path=html_text(item.get("path", "")), + pattern=html_text(item.get("source_pattern", "")), + sha=html_text(item.get("sha256") or "n/a"), + ) + for item in items + ) + + def render_html_item(item: dict[str, Any]) -> str: must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} return f""" @@ -218,14 +363,18 @@ def render_html_item(item: dict[str, Any]) -> str: def render_html(report: dict[str, Any]) -> str: summary = report["summary"] + artifact_ready = summary.get("artifact_ready_count", 0) + artifact_total = summary.get("artifact_checklist_count", 0) stats = [ ("Requested", summary["requested_count"]), ("Written", summary["written_count"]), ("Existing", summary["existing_count"]), ("Skipped", summary["skipped_count"]), + ("Artifacts", f"{artifact_ready}/{artifact_total}"), ] stat_html = "".join(f"
{html_text(label)}{html_text(value)}
" for label, value in stats) evidence_html = "".join(render_html_item(item) for item in report.get("evidence_items", [])) + artifact_html = render_html_artifact_checklist(report.get("artifact_checklist", [])) return f""" @@ -250,7 +399,7 @@ def render_html(report: dict[str, Any]) -> str: h3 {{ margin:4px 0 10px; font-size:22px; }} h4 {{ margin:0 0 8px; font-size:16px; }} .lede {{ max-width:800px; color:var(--muted); font-size:20px; }} - .stats {{ display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin:26px 0 0; }} + .stats {{ display:grid; grid-template-columns:repeat(5, minmax(0,1fr)); gap:12px; margin:26px 0 0; }} .stats article, .panel, .draft-card, .evidence-card {{ border:1px solid var(--line); border-radius:8px; background:#fff; }} .stats article {{ padding:16px; }} .stats span, .draft-card span, .evidence-card span, .muted {{ color:var(--muted); }} @@ -258,11 +407,12 @@ def render_html(report: dict[str, Any]) -> str: .section {{ padding:32px 0; border-bottom:1px solid var(--line); }} .panel {{ padding:20px; }} .two-col {{ display:grid; grid-template-columns:minmax(0, .45fr) minmax(0, 1fr); gap:18px; align-items:start; }} - .draft-grid, .evidence-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} - .draft-card, .evidence-card {{ padding:18px; min-width:0; }} + .draft-grid, .evidence-grid, .artifact-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} + .draft-card, .evidence-card, .artifact-card {{ padding:18px; min-width:0; }} .draft-card.written, .draft-card.exists {{ border-left:4px solid var(--pass); }} .draft-card.skipped {{ border-left:4px solid var(--warn); }} - .evidence-card.awaiting-submission, .evidence-card.fix-submission, .evidence-card.fix-template {{ border-left:4px solid var(--warn); }} + .evidence-card.awaiting-submission, .evidence-card.fix-submission, .evidence-card.fix-template, .artifact-card.missing, .artifact-card.glob-no-match, .artifact-card.unsafe-path, .artifact-card.raw-content-disallowed {{ border-left:4px solid var(--warn); }} + .artifact-card.ready {{ border-left:4px solid var(--pass); }} dl {{ display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }} dt {{ color:var(--ink); }} dd {{ margin:0; min-width:0; overflow-wrap:anywhere; }} @@ -276,11 +426,11 @@ def render_html(report: dict[str, Any]) -> str: .mini-grid li, .notice li {{ overflow-wrap:anywhere; }} .notice {{ background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; }} .errors {{ color:var(--warn); }} - @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} + @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} - +
Evidence Intake @@ -293,6 +443,7 @@ def render_html(report: dict[str, Any]) -> str:

Drafts

{render_html_files(report['files'])}
+

Artifact Checklist

Copy concrete paths and SHA-256 digests from here into artifact_refs after real evidence exists. Glob patterns are expanded for operator convenience only.

{artifact_html}

Evidence Requirements

{evidence_html}

Safety Boundary

  • Drafts never count as accepted ledger evidence.
  • Valid intake means ready for ledger review, not world-class completion.
  • Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content.
@@ -315,12 +466,16 @@ def build_submission_kit( unknown_keys = sorted(set(evidence_keys or []) - valid_keys) template_results = template_result_by_key(intake) files = [copy_template(skill_dir, output_dir, item, template_results, overwrite) for item in items] + artifact_checklist = build_artifact_checklist(skill_dir, items) manifest_path = output_dir / "submission_manifest.json" readme_path = output_dir / "README.md" output_html = output_html or (output_dir / "index.html") written_count = sum(1 for item in files if item["status"] == "written") existing_count = sum(1 for item in files if item["status"] == "exists") skipped_count = sum(1 for item in files if item["status"] == "skipped") + artifact_ready_count = sum(1 for item in artifact_checklist if item.get("artifact_ref_ready")) + artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready")) + artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required")) ok = not unknown_keys and skipped_count == 0 report = { "schema_version": "1.0", @@ -335,12 +490,17 @@ def build_submission_kit( "existing_count": existing_count, "skipped_count": skipped_count, "unknown_key_count": len(unknown_keys), + "artifact_checklist_count": len(artifact_checklist), + "artifact_ready_count": artifact_ready_count, + "artifact_missing_count": artifact_missing_count, + "artifact_glob_expansion_count": artifact_glob_count, "drafts_count_as_evidence": False, "ledger_counts_submission_as_completion": False, "decision": "submission-kit-ready" if ok else "fix-submission-kit", }, "unknown_evidence_keys": unknown_keys, "files": files, + "artifact_checklist": artifact_checklist, "evidence_items": items, "commands": { "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {shell_path(output_dir, skill_dir)}", diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 5e37ea9..9d9e5fd 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -189,25 +189,48 @@ def main() -> None: assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"] assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"] assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"] assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"] assert kit_payload["files"][0]["output_path"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/provider-holdout.json"), kit_payload["files"] + artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]} + assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows + assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) assert kit_draft["template_only"] is True, kit_draft assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8")) assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"] + assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"] assert kit_manifest["artifacts"]["html"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/index.html"), kit_manifest["artifacts"] kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8") assert "Drafts are not accepted evidence." in kit_readme, kit_readme assert "validate intake" in kit_readme, kit_readme + assert "Artifact Checklist" in kit_readme, kit_readme + assert "reports/output_execution_runs.json" in kit_readme, kit_readme kit_html = (kit_dir / "index.html").read_text(encoding="utf-8") assert "World-Class Evidence Submission Kit" in kit_html, kit_html assert "Drafts are not accepted evidence" in kit_html, kit_html assert "provider-holdout" in kit_html, kit_html + assert "Artifact Checklist" in kit_html, kit_html assert "World-Class Evidence Submission Kit" in kit_html, kit_html assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html + + native_kit_dir = TMP / "native_permission_kit" + native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement") + native_kit_payload = json.loads(native_kit_proc.stdout) + native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]} + assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 4, native_kit_payload["summary"] + assert "dist/targets/openai/adapter.json" in native_rows, native_rows + assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows + assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows + assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows + native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8") + assert "Glob patterns are expanded into concrete files" in native_readme, native_readme draft_intake = run_intake("--submissions-dir", str(kit_dir)) assert draft_intake["ok"] is False, draft_intake assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"]