From 00ee3a61aa2de9e0a304c46ffa348c657ce7d220 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 22:24:13 +0800 Subject: [PATCH] Add world-class submission handoff steps --- reports/context_budget.json | 14 +-- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 8 +- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- scripts/prepare_world_class_submission_kit.py | 111 +++++++++++++++++- .../world_class_submission_kit_rendering.py | 62 +++++++++- tests/verify_world_class_submission_kit.py | 29 +++++ 8 files changed, 206 insertions(+), 24 deletions(-) diff --git a/reports/context_budget.json b/reports/context_budget.json index 8c80286..cf759b3 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1053172, + "other_text_tokens": 1055458, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1053969, - "deferred_resource_tokens": 487673, + "estimated_total_text_tokens": 1056255, + "deferred_resource_tokens": 489507, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 427575, + "estimated_tokens": 429409, "file_count": 135 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 427575, + "estimated_tokens": 429409, "file_count": 135 } ], @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 427575, + "estimated_tokens": 429409, "file_count": 135, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 662, + "relevant_file_count": 663, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index ed70295..312a95b 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 487673 | `governed` | scripts:427575 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 489507 | `governed` | scripts:429409 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 5f9e288..f72433c 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-16", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 487673, + "deferred_resource_tokens": 489507, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 427575, + "estimated_tokens": 429409, "file_count": 135 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 427575, + "estimated_tokens": 429409, "file_count": 135, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index c1041e2..2bdc0ee 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 226, - "package_sha256": "11432d13467e6822ac0fff5d8bde27fdf326322dfed62aca7cb9ad5ee1a706ab" + "package_sha256": "c65410d8e0bb1ea55fc52ea0b2c557f9590b232722b51e77e059595f64bf668f" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 5a305e3..7a650bc 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `226` -- Package SHA256: `11432d13467e6822ac0fff5d8bde27fdf326322dfed62aca7cb9ad5ee1a706ab` +- Package SHA256: `c65410d8e0bb1ea55fc52ea0b2c557f9590b232722b51e77e059595f64bf668f` ## Failures diff --git a/scripts/prepare_world_class_submission_kit.py b/scripts/prepare_world_class_submission_kit.py index ae28c9f..abe15fb 100644 --- a/scripts/prepare_world_class_submission_kit.py +++ b/scripts/prepare_world_class_submission_kit.py @@ -299,6 +299,85 @@ def build_artifact_checklist(skill_dir: Path, items: list[dict[str, Any]]) -> li return rows +def build_handoff_steps( + commands: dict[str, str], + *, + files: list[dict[str, Any]], + source_blocked_count: int, + invalid_draft_count: int, +) -> list[dict[str, Any]]: + drafts_ready = bool(files) and invalid_draft_count == 0 + return [ + { + "step_id": "prepare-drafts", + "label": "Prepare editable drafts", + "status": "ready" if drafts_ready else "fix-required", + "command": commands["prepare_submission"], + "completion_signal": "JSON drafts, submission_manifest.json, README.md, and index.html are present.", + "counts_as_completion": False, + "blocking_condition": "One or more drafts were skipped or failed template validation." if invalid_draft_count else "", + }, + { + "step_id": "collect-source", + "label": "Collect real external or human evidence", + "status": "blocked" if source_blocked_count else "ready", + "command": "", + "completion_signal": "Source aggregate reports satisfy the required provider, human, native, or telemetry checks.", + "counts_as_completion": False, + "blocking_condition": ( + f"{source_blocked_count} source check(s) still block ledger review." + if source_blocked_count + else "" + ), + }, + { + "step_id": "edit-submission", + "label": "Edit submission packet", + "status": "manual", + "command": "", + "completion_signal": "template_only is false only after real evidence exists and all attestation fields are truthful.", + "counts_as_completion": False, + "blocking_condition": "Raw prompts, outputs, transcripts, credentials, or private content must not be included.", + }, + { + "step_id": "validate-intake", + "label": "Validate intake contract", + "status": "pending", + "command": commands["validate_intake"], + "completion_signal": "world_class_evidence_intake reports valid submissions and no invalid packets.", + "counts_as_completion": False, + "blocking_condition": "A valid packet is ready for ledger review but is not accepted evidence by itself.", + }, + { + "step_id": "review-submission", + "label": "Review submission queue", + "status": "pending", + "command": commands["review_submission"], + "completion_signal": "world_class_submission_review shows ready-for-ledger-review for the submitted key.", + "counts_as_completion": False, + "blocking_condition": "Reviewer queue output is advisory and cannot accept evidence.", + }, + { + "step_id": "refresh-ledger", + "label": "Refresh evidence ledger", + "status": "pending", + "command": commands["refresh_ledger"], + "completion_signal": "world_class_evidence_ledger accepts the evidence entry with valid source checks.", + "counts_as_completion": False, + "blocking_condition": "Ledger remains the only world-class readiness source of truth.", + }, + { + "step_id": "guard-claim", + "label": "Guard public claim", + "status": "pending", + "command": commands["guard_claim"], + "completion_signal": "world_class_claim_guard allows the public readiness claim.", + "counts_as_completion": False, + "blocking_condition": "Public world-class claims stay blocked until every ledger entry is accepted.", + }, + ] + + def build_submission_kit( skill_dir: Path, output_dir: Path, @@ -347,6 +426,26 @@ def build_submission_kit( source_summary = summarize_source_checklist(source_checklist) matrix_summary = summarize_evidence_matrix(evidence_matrix) ok = not unknown_keys and skipped_count == 0 + output_dir_arg = shell_path(output_dir, skill_dir) + requested_key_args = " ".join(f"--evidence-key {shlex.quote(key)}" for key in (evidence_keys or [])) + prepare_command = f"python3 scripts/yao.py world-class-submission-kit . --output-dir {output_dir_arg}" + if requested_key_args: + prepare_command = f"{prepare_command} {requested_key_args}" + if prefill_artifacts: + prepare_command = f"{prepare_command} --prefill-artifacts" + commands = { + "prepare_submission": prepare_command, + "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {output_dir_arg}", + "review_submission": f"python3 scripts/yao.py world-class-submission-review . --submissions-dir {output_dir_arg}", + "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {output_dir_arg}", + "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", + } + handoff_steps = build_handoff_steps( + commands, + files=files, + source_blocked_count=source_summary["source_blocked_count"], + invalid_draft_count=skipped_count + len(unknown_keys), + ) report = { "schema_version": "1.0", "ok": ok, @@ -375,6 +474,10 @@ def build_submission_kit( "artifact_ref_unfilled_count": unfilled_artifact_ref_count, **source_summary, **matrix_summary, + "handoff_step_count": len(handoff_steps), + "handoff_blocked_count": sum(1 for item in handoff_steps if item["status"] == "blocked"), + "handoff_fix_required_count": sum(1 for item in handoff_steps if item["status"] == "fix-required"), + "handoff_counts_as_completion": False, "drafts_count_as_evidence": False, "ledger_counts_submission_as_completion": False, "decision": "submission-kit-ready" if ok else "fix-submission-kit", @@ -384,15 +487,13 @@ def build_submission_kit( "artifact_checklist": artifact_checklist, "source_checklist": source_checklist, "evidence_matrix": evidence_matrix, + "operator_handoff": handoff_steps, "evidence_items": items, - "commands": { - "validate_intake": f"python3 scripts/yao.py world-class-intake . --submissions-dir {shell_path(output_dir, skill_dir)}", - "refresh_ledger": f"python3 scripts/yao.py world-class-ledger . --submissions-dir {shell_path(output_dir, skill_dir)}", - "guard_claim": "python3 scripts/yao.py world-class-claim-guard .", - }, + "commands": commands, "safety": { "template_only_drafts": True, "artifact_prefill_counts_as_evidence": False, + "operator_handoff_counts_as_evidence": False, "real_evidence_required_before_template_only_false": True, "raw_content_allowed": False, "credentials_allowed": False, diff --git a/scripts/world_class_submission_kit_rendering.py b/scripts/world_class_submission_kit_rendering.py index 3107474..09024fb 100644 --- a/scripts/world_class_submission_kit_rendering.py +++ b/scripts/world_class_submission_kit_rendering.py @@ -31,14 +31,31 @@ def render_readme(report: dict[str, Any]) -> str: "## Commands", "", f"- validate intake: `{commands['validate_intake']}`", + f"- review submission: `{commands['review_submission']}`", f"- refresh ledger: `{commands['refresh_ledger']}`", f"- guard public claims: `{commands['guard_claim']}`", "", + "## Operator Handoff", + "", + "Follow these steps in order when handing the kit from operator to reviewer. Handoff rows are procedural and never count as completion evidence.", + "", + "| Step | Status | Command | Completion signal |", + "| --- | --- | --- | --- |", + ] + for item in report.get("operator_handoff", []): + command = item.get("command") or "manual" + lines.append( + f"| `{item['step_id']}` | `{item['status']}` | `{command}` | {item['completion_signal']} |" + ) + lines.extend( + [ + "", "## Drafts", "", "| Evidence | Draft | Status | Prefilled refs |", "| --- | --- | --- | ---: |", - ] + ] + ) for item in report["files"]: lines.append( f"| `{item['evidence_key']}` | `{item['output_path']}` | `{item['status']}` | `{item.get('prefilled_artifact_ref_count', 0)}` |" @@ -259,6 +276,37 @@ def render_html_matrix(items: list[dict[str, Any]]) -> str: ) +def render_html_handoff(items: list[dict[str, Any]]) -> str: + if not items: + return '

No operator handoff steps were generated.

' + return "".join( + """ +
+
+ {status} +

{label}

+
+
+
Step
{step}
+
Command
{command}
+
Signal
{signal}
+
Evidence
{counts}
+
+

{blocking}

+
+ """.format( + status=html_text(item.get("status", "")), + label=html_text(item.get("label", "")), + step=html_text(item.get("step_id", "")), + command=html_text(item.get("command") or "manual"), + signal=html_text(item.get("completion_signal", "")), + counts="does not count as completion" if item.get("counts_as_completion") is False else "review required", + blocking=html_text(item.get("blocking_condition", "")), + ) + for item in items + ) + + def render_html_item(item: dict[str, Any]) -> str: must_collect = item.get("must_collect", {}) if isinstance(item.get("must_collect", {}), dict) else {} runbook = must_collect.get("runbook", []) @@ -317,6 +365,7 @@ def render_html(report: dict[str, Any]) -> str: ) evidence_html = "".join(render_html_item(item) for item in report.get("evidence_items", [])) matrix_html = render_html_matrix(report.get("evidence_matrix", [])) + handoff_html = render_html_handoff(report.get("operator_handoff", [])) artifact_html = render_html_artifact_checklist(report.get("artifact_checklist", [])) source_html = render_html_source_checklist(report.get("source_checklist", [])) return f""" @@ -351,12 +400,14 @@ def render_html(report: dict[str, Any]) -> str: .section {{ padding:32px 0; border-bottom:1px solid var(--line); }} .panel {{ padding:20px; }} .two-col {{ display:grid; grid-template-columns:minmax(0, .45fr) minmax(0, 1fr); gap:18px; align-items:start; }} - .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .matrix-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} - .draft-card, .evidence-card, .artifact-card, .source-card, .matrix-card {{ padding:18px; min-width:0; border:1px solid var(--line); border-radius:8px; background:#fff; }} + .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .matrix-grid, .handoff-grid {{ display:grid; grid-template-columns:repeat(2, minmax(0,1fr)); gap:16px; }} + .draft-card, .evidence-card, .artifact-card, .source-card, .matrix-card, .handoff-card {{ padding:18px; min-width:0; border:1px solid var(--line); border-radius:8px; background:#fff; }} .draft-card.written, .draft-card.exists {{ border-left:4px solid var(--pass); }} .draft-card.skipped {{ border-left:4px solid var(--warn); }} .matrix-card.collect-source, .matrix-card.prepare-draft, .matrix-card.fix-artifacts, .matrix-card.fix-draft {{ border-left:4px solid var(--warn); }} .matrix-card.validate-packet {{ border-left:4px solid var(--pass); }} + .handoff-card.blocked, .handoff-card.fix-required {{ border-left:4px solid var(--warn); }} + .handoff-card.ready {{ border-left:4px solid var(--pass); }} .evidence-card.awaiting-submission, .evidence-card.fix-submission, .evidence-card.fix-template, .artifact-card.missing, .artifact-card.glob-no-match, .artifact-card.unsafe-path, .artifact-card.raw-content-disallowed {{ border-left:4px solid var(--warn); }} .artifact-card.ready {{ border-left:4px solid var(--pass); }} .source-card.blocked {{ border-left:4px solid var(--warn); }} @@ -374,11 +425,11 @@ def render_html(report: dict[str, Any]) -> str: .mini-grid li, .runbook-panel li, .notice li {{ overflow-wrap:anywhere; }} .notice {{ background:var(--soft); border-left:4px solid var(--ink); padding:16px; border-radius:8px; }} .errors {{ color:var(--warn); }} - @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .matrix-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} + @media (max-width:820px) {{ .stats, .two-col, .draft-grid, .evidence-grid, .artifact-grid, .source-grid, .matrix-grid, .handoff-grid, .mini-grid {{ grid-template-columns:1fr; }} h1 {{ font-size:38px; }} .topbar-inner {{ align-items:flex-start; flex-direction:column; }} }} - +
Evidence Intake @@ -390,6 +441,7 @@ def render_html(report: dict[str, Any]) -> str:

Workflow

  1. Run the real provider, human review, native permission, or native client telemetry work first.
  2. Edit the matching JSON draft with aggregate artifact references and provenance metadata.
  3. Set template_only to false only after real evidence exists.
  4. Use prefilled SHA-256 values as convenience data, not evidence acceptance.
  5. Validate intake, refresh the ledger, then guard public claims.
+

Operator Handoff

These ordered steps make the operator-to-reviewer handoff auditable. They are procedural guardrails and never count as world-class evidence.

{handoff_html}

Evidence Matrix

The matrix separates submission artifact_refs from supporting assets, then combines draft status, source checks, and the next operator action. It is guidance only and never counts as accepted evidence.

{matrix_html}

Drafts

{render_html_files(report['files'])}

Artifact Checklist

Rows marked submission-ref are the paths expected in artifact_refs. Supporting-evidence rows help reviewers audit the packet but do not all need to be copied into the submission. Glob patterns are expanded for operator convenience only.

{artifact_html}
diff --git a/tests/verify_world_class_submission_kit.py b/tests/verify_world_class_submission_kit.py index 60fc579..fd765fb 100644 --- a/tests/verify_world_class_submission_kit.py +++ b/tests/verify_world_class_submission_kit.py @@ -97,9 +97,15 @@ def main() -> None: assert kit_payload["summary"]["evidence_matrix_supporting_artifact_count"] >= 1, kit_payload["summary"] assert kit_payload["summary"]["evidence_matrix_supporting_artifact_ready_count"] >= 1, kit_payload["summary"] assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["handoff_step_count"] == 7, kit_payload["summary"] + assert kit_payload["summary"]["handoff_blocked_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["handoff_fix_required_count"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["handoff_counts_as_completion"] is False, kit_payload["summary"] assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"] + assert kit_payload["safety"]["operator_handoff_counts_as_evidence"] is False, kit_payload["safety"] + assert "review_submission" in kit_payload["commands"], kit_payload["commands"] assert kit_payload["files"][0]["output_path"].endswith( f"tests/{TMP.name}/submission_kit/provider-holdout.json" ), kit_payload["files"] @@ -121,6 +127,23 @@ def main() -> None: assert matrix_row["counts_as_completion"] is False, matrix_row assert "real credentials" in matrix_row["next_action"], matrix_row + handoff_steps = {item["step_id"]: item for item in kit_payload["operator_handoff"]} + assert list(handoff_steps) == [ + "prepare-drafts", + "collect-source", + "edit-submission", + "validate-intake", + "review-submission", + "refresh-ledger", + "guard-claim", + ], handoff_steps + assert handoff_steps["prepare-drafts"]["status"] == "ready", handoff_steps + assert handoff_steps["collect-source"]["status"] == "blocked", handoff_steps + assert handoff_steps["collect-source"]["counts_as_completion"] is False, handoff_steps + assert "source check" in handoff_steps["collect-source"]["blocking_condition"], handoff_steps + assert "world-class-submission-review" in handoff_steps["review-submission"]["command"], handoff_steps + assert all(item["counts_as_completion"] is False for item in kit_payload["operator_handoff"]), handoff_steps + artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]} assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows @@ -165,6 +188,9 @@ def main() -> None: assert "validate intake" in kit_readme, kit_readme assert "Artifact Checklist" in kit_readme, kit_readme assert "Evidence Matrix" in kit_readme, kit_readme + assert "Operator Handoff" in kit_readme, kit_readme + assert "Handoff rows are procedural" in kit_readme, kit_readme + assert "`review-submission`" in kit_readme, kit_readme assert "Submission refs" in kit_readme, kit_readme assert "Supporting assets" in kit_readme, kit_readme assert "`submission-ref`" in kit_readme, kit_readme @@ -181,6 +207,9 @@ def main() -> None: assert "provider-holdout" in kit_html, kit_html assert "Artifact Checklist" in kit_html, kit_html assert "Evidence Matrix" in kit_html, kit_html + assert "Operator Handoff" in kit_html, kit_html + assert "handoff-card blocked" in kit_html, kit_html + assert "does not count as completion" in kit_html, kit_html assert "matrix-card collect-source" in kit_html, kit_html assert ( f"
Submission refs
{matrix_row['submission_ref_ready_count']}/{matrix_row['submission_ref_total_count']} ready
"