frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: add world-class submission artifact checklist

Browse Source
This commit is contained in:
yaojingang
2026-06-14 13:07:12 +08:00
parent e83ee842b2
commit 6fc7a130ce
2 changed files with 189 additions and 6 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/prepare_world_class_submission_kit.py
+166 -6
View File
@@ -1,5 +1,6 @@
#!/usr/bin/env python3
import argparse
import hashlib
import html
import json
import shlex
@@ -9,6 +10,7 @@ from pathlib import Path
from typing import Any
from render_world_class_evidence_intake import build_intake
from world_class_evidence_contract import DISALLOWED_REAL_ARTIFACTS
ROOT = Path(__file__).resolve().parent.parent
@@ -36,6 +38,18 @@ def html_text(value: Any) -> str:
return html.escape(str(value or ""), quote=True)
def has_glob_pattern(value: str) -> bool:
return any(token in value for token in ("*", "?", "["))
def sha256_file(path: Path) -> str:
digest = hashlib.sha256()
with path.open("rb") as handle:
for chunk in iter(lambda: handle.read(1024 * 1024), b""):
digest.update(chunk)
return digest.hexdigest()
def requested_checklist_items(intake: dict[str, Any], evidence_keys: list[str]) -> list[dict[str, Any]]:
items = intake.get("operator_checklist", [])
if not evidence_keys:
@@ -95,6 +109,94 @@ def copy_template(
}
def artifact_row(skill_dir: Path, evidence_key: str, pattern: str, path: Path, status: str) -> dict[str, Any]:
exists = path.exists()
is_file = exists and path.is_file()
relative = rel_path(path, skill_dir) if exists else pattern
contains_raw_content = relative in DISALLOWED_REAL_ARTIFACTS
digest = sha256_file(path) if is_file and not contains_raw_content else ""
row_status = status if status else ("ready" if is_file else "missing")
if contains_raw_content:
row_status = "raw-content-disallowed"
return {
"evidence_key": evidence_key,
"source_pattern": pattern,
"path": relative,
"status": row_status,
"exists": exists,
"is_file": is_file,
"sha256": digest,
"artifact_ref_ready": bool(is_file and digest and not contains_raw_content),
"copy_path": relative if is_file else "",
"copy_sha256": digest,
"contains_raw_content": contains_raw_content,
"concrete_reference_required": has_glob_pattern(pattern),
}
def artifact_checklist_for_item(skill_dir: Path, item: dict[str, Any]) -> list[dict[str, Any]]:
key = str(item.get("evidence_key", ""))
must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {}
artifacts = must_collect.get("evidence_artifacts", [])
if not isinstance(artifacts, list):
return []
rows: list[dict[str, Any]] = []
for artifact in artifacts:
pattern = str(artifact or "").strip()
if not pattern:
continue
if Path(pattern).is_absolute() or ".." in Path(pattern).parts:
rows.append(
{
"evidence_key": key,
"source_pattern": pattern,
"path": pattern,
"status": "unsafe-path",
"exists": False,
"is_file": False,
"sha256": "",
"artifact_ref_ready": False,
"copy_path": "",
"copy_sha256": "",
"contains_raw_content": False,
"concrete_reference_required": True,
}
)
continue
if has_glob_pattern(pattern):
matches = sorted(path for path in skill_dir.glob(pattern) if path.is_file())
if not matches:
rows.append(
{
"evidence_key": key,
"source_pattern": pattern,
"path": pattern,
"status": "glob-no-match",
"exists": False,
"is_file": False,
"sha256": "",
"artifact_ref_ready": False,
"copy_path": "",
"copy_sha256": "",
"contains_raw_content": False,
"concrete_reference_required": True,
}
)
continue
for match in matches:
rows.append(artifact_row(skill_dir, key, pattern, match, "ready"))
continue
rows.append(artifact_row(skill_dir, key, pattern, skill_dir / pattern, ""))
return rows
def build_artifact_checklist(skill_dir: Path, items: list[dict[str, Any]]) -> list[dict[str, Any]]:
rows: list[dict[str, Any]] = []
for item in items:
rows.extend(artifact_checklist_for_item(skill_dir, item))
return rows
def render_readme(report: dict[str, Any]) -> str:
commands = report["commands"]
lines = [
@@ -125,6 +227,22 @@ def render_readme(report: dict[str, Any]) -> str:
]
for item in report["files"]:
lines.append(f"| `{item['evidence_key']}` | `{item['output_path']}` | `{item['status']}` |")
lines.extend(
[
"",
"## Artifact Checklist",
"",
"Use these paths and SHA-256 digests when filling `artifact_refs`. Glob patterns are expanded into concrete files; submissions must reference concrete paths, not globs.",
"",
"| Evidence | Path | Status | SHA-256 |",
"| --- | --- | --- | --- |",
]
)
for item in report.get("artifact_checklist", []):
digest = item.get("sha256") or "n/a"
lines.append(
f"| `{item['evidence_key']}` | `{item['path']}` | `{item['status']}` | `{digest}` |"
)
lines.extend(
[
"",
@@ -184,6 +302,33 @@ def render_html_files(files: list[dict[str, Any]]) -> str:
)
def render_html_artifact_checklist(items: list[dict[str, Any]]) -> str:
if not items:
return "<p class=\"muted\">No required artifacts were listed for the requested evidence.</p>"
return "".join(
"""
<article class="artifact-card {status}">
<div>
<span>{key}</span>
<h3>{path}</h3>
</div>
<dl>
<dt>Pattern</dt><dd><code>{pattern}</code></dd>
<dt>Status</dt><dd>{status}</dd>
<dt>SHA-256</dt><dd><code>{sha}</code></dd>
</dl>
</article>
""".format(
status=html_text(item.get("status", "")),
key=html_text(item.get("evidence_key", "")),
path=html_text(item.get("path", "")),
pattern=html_text(item.get("source_pattern", "")),
sha=html_text(item.get("sha256") or "n/a"),
)
for item in items
)
def render_html_item(item: dict[str, Any]) -> str:
must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {}
return f"""
@@ -218,14 +363,18 @@ def render_html_item(item: dict[str, Any]) -> str:
def render_html(report: dict[str, Any]) -> str:
summary = report["summary"]
artifact_ready = summary.get("artifact_ready_count", 0)
artifact_total = summary.get("artifact_checklist_count", 0)
stats = [
("Requested", summary["requested_count"]),
("Written", summary["written_count"]),
("Existing", summary["existing_count"]),
("Skipped", summary["skipped_count"]),
("Artifacts", f"{artifact_ready}/{artifact_total}"),
]
stat_html = "".join(f"<article><span>{html_text(label)}</span><strong>{html_text(value)}</strong></article>" for label, value in stats)
evidence_html = "".join(render_html_item(item) for item in report.get("evidence_items", []))
artifact_html = render_html_artifact_checklist(report.get("artifact_checklist", []))
return f"""<!doctype html>
<html lang="en">
<head>
@@ -250,7 +399,7 @@ def render_html(report: dict[str, Any]) -> str:
h3 {{ margin:4px 0 10px; font-size:22px; }}
h4 {{ margin:0 0 8px; font-size:16px; }}
.lede {{ max-width:800px; color:var(--muted); font-size:20px; }}
.stats {{ display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin:26px 0 0; }}
.stats {{ display:grid; grid-template-columns:repeat(5, minmax(0,1fr)); gap:12px; margin:26px 0 0; }}
.stats article, .panel, .draft-card, .evidence-card {{ border:1px solid var(--line); border-radius:8px; background:#fff; }}
.stats article {{ padding:16px; }}
.stats span, .draft-card span, .evidence-card span, .muted {{ color:var(--muted); }}
@@ -258,11 +407,12 @@ def render_html(report: dict[str, Any]) -> str:
.section {{ padding:32px 0; border-bottom:1px solid var(--line); }}
.panel {{ padding:20px; }}
.two-col {{ display:grid; grid-template-columns:minmax(0, .45fr) minmax(0, 1fr); gap:18px; align-items:start; }}
.draft-grid, .evidence-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }}
.draft-card, .evidence-card {{ padding:18px; min-width:0; }}
.draft-grid, .evidence-grid, .artifact-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }}
.draft-card, .evidence-card, .artifact-card {{ padding:18px; min-width:0; }}
.draft-card.written, .draft-card.exists {{ border-left:4px solid var(--pass); }}
.draft-card.skipped {{ border-left:4px solid var(--warn); }}
.evidence-card.awaiting-submission, .evidence-card.fix-submission, .evidence-card.fix-template {{ border-left:4px solid var(--warn); }}
.evidence-card.awaiting-submission, .evidence-card.fix-submission, .evidence-card.fix-template, .artifact-card.missing, .artifact-card.glob-no-match, .artifact-card.unsafe-path, .artifact-card.raw-content-disallowed {{ border-left:4px solid var(--warn); }}
.artifact-card.ready {{ border-left:4px solid var(--pass); }}
dl {{ display:grid; grid-template-columns:96px minmax(0,1fr); gap:8px 12px; }}
dt {{ color:var(--ink); }}
dd {{ margin:0; min-width:0; overflow-wrap:anywhere; }}
@@ -276,11 +426,11 @@ def render_html(report: dict[str, Any]) -> str:
.mini-grid li, .notice li {{ overflow-wrap:anywhere; }}
.notice {{ background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; }}
.errors {{ color:var(--warn); }}
@media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }}
@media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }}
</style>
</head>
<body>
<nav class="topbar"><div class="topbar-inner"><span class="brand">World-Class Kit</span><div class="links"><a href="#workflow">Workflow</a><a href="#drafts">Drafts</a><a href="#evidence">Evidence</a><a href="#safety">Safety</a></div></div></nav>
<nav class="topbar"><div class="topbar-inner"><span class="brand">World-Class Kit</span><div class="links"><a href="#workflow">Workflow</a><a href="#drafts">Drafts</a><a href="#artifacts">Artifacts</a><a href="#evidence">Evidence</a><a href="#safety">Safety</a></div></div></nav>
<main class="shell">
<section class="hero">
<span class="eyebrow">Evidence Intake</span>
@@ -293,6 +443,7 @@ def render_html(report: dict[str, Any]) -> str:
<aside class="panel"><h2>Commands</h2><ul class="commands">{render_html_commands(report['commands'])}</ul></aside>
</section>
<section class="section" id="drafts"><h2>Drafts</h2><div class="draft-grid">{render_html_files(report['files'])}</div></section>
<section class="section" id="artifacts"><h2>Artifact Checklist</h2><p class="muted">Copy concrete paths and SHA-256 digests from here into artifact_refs after real evidence exists. Glob patterns are expanded for operator convenience only.</p><div class="artifact-grid">{artifact_html}</div></section>
<section class="section" id="evidence"><h2>Evidence Requirements</h2><div class="evidence-grid">{evidence_html}</div></section>
<section class="section" id="safety"><h2>Safety Boundary</h2><div class="notice"><ul><li>Drafts never count as accepted ledger evidence.</li><li>Valid intake means ready for ledger review, not world-class completion.</li><li>Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content.</li></ul></div></section>
</main>
@@ -315,12 +466,16 @@ def build_submission_kit(
unknown_keys = sorted(set(evidence_keys or []) - valid_keys)
template_results = template_result_by_key(intake)
files = [copy_template(skill_dir, output_dir, item, template_results, overwrite) for item in items]
artifact_checklist = build_artifact_checklist(skill_dir, items)
manifest_path = output_dir / "submission_manifest.json"
readme_path = output_dir / "README.md"
output_html = output_html or (output_dir / "index.html")
written_count = sum(1 for item in files if item["status"] == "written")
existing_count = sum(1 for item in files if item["status"] == "exists")
skipped_count = sum(1 for item in files if item["status"] == "skipped")
artifact_ready_count = sum(1 for item in artifact_checklist if item.get("artifact_ref_ready"))
artifact_missing_count = sum(1 for item in artifact_checklist if not item.get("artifact_ref_ready"))
artifact_glob_count = sum(1 for item in artifact_checklist if item.get("concrete_reference_required"))
ok = not unknown_keys and skipped_count == 0
report = {
"schema_version": "1.0",
@@ -335,12 +490,17 @@ def build_submission_kit(
"existing_count": existing_count,
"skipped_count": skipped_count,
"unknown_key_count": len(unknown_keys),
"artifact_checklist_count": len(artifact_checklist),
"artifact_ready_count": artifact_ready_count,
"artifact_missing_count": artifact_missing_count,
"artifact_glob_expansion_count": artifact_glob_count,
"drafts_count_as_evidence": False,
"ledger_counts_submission_as_completion": False,
"decision": "submission-kit-ready" if ok else "fix-submission-kit",
},
"unknown_evidence_keys": unknown_keys,
"files": files,
"artifact_checklist": artifact_checklist,
"evidence_items": items,
"commands": {
"validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {shell_path(output_dir, skill_dir)}",
tests/verify_world_class_evidence_intake.py
+23
View File
@@ -189,25 +189,48 @@ def main() -> None:
assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"]
assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"]
assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"]
assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"]
assert kit_payload["files"][0]["output_path"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/provider-holdout.json"), kit_payload["files"]
artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]}
assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows
assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows
kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8"))
assert kit_draft["template_only"] is True, kit_draft
assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft
kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8"))
assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"]
assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"]
assert kit_manifest["artifacts"]["html"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/index.html"), kit_manifest["artifacts"]
kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8")
assert "Drafts are not accepted evidence." in kit_readme, kit_readme
assert "validate intake" in kit_readme, kit_readme
assert "Artifact Checklist" in kit_readme, kit_readme
assert "reports/output_execution_runs.json" in kit_readme, kit_readme
kit_html = (kit_dir / "index.html").read_text(encoding="utf-8")
assert "<title>World-Class Evidence Submission Kit</title>" in kit_html, kit_html
assert "Drafts are not accepted evidence" in kit_html, kit_html
assert "provider-holdout" in kit_html, kit_html
assert "Artifact Checklist" in kit_html, kit_html
assert "World-Class Evidence Submission Kit" in kit_html, kit_html
assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html
native_kit_dir = TMP / "native_permission_kit"
native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement")
native_kit_payload = json.loads(native_kit_proc.stdout)
native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]}
assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 4, native_kit_payload["summary"]
assert "dist/targets/openai/adapter.json" in native_rows, native_rows
assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows
assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows
assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows
native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8")
assert "Glob patterns are expanded into concrete files" in native_readme, native_readme
draft_intake = run_intake("--submissions-dir", str(kit_dir))
assert draft_intake["ok"] is False, draft_intake
assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"]