diff --git a/AGENTS.md b/AGENTS.md index 7190529e..9ddab014 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -76,6 +76,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete. - `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence. - `scripts/render_world_class_evidence_intake.py`: intake validator for external and human world-class evidence packets. Real submissions must reference concrete local aggregate artifacts with matching SHA-256 digests; templates may stay hash-free and must not count as evidence. +- `scripts/world_class_evidence_contract.py`: shared intake contract and artifact-integrity validator. Keep ledger, intake, and submission review aligned so source evidence cannot be accepted without a valid real submission and matching artifact SHA-256 checks. - `scripts/render_world_class_submission_review.py`: read-only queue for external and human evidence packets after intake validation. Keep it from accepting evidence; it may only compare packet validity, source evidence checks, and ledger state. - `scripts/render_world_class_operator_runbook.py`: operator-facing world-class evidence runbook. Keep it as coordination guidance only; it must not accept evidence or flip world-class readiness. - `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable. diff --git a/registry/index.json b/registry/index.json index 1202c03d..58dc5ab3 100644 --- a/registry/index.json +++ b/registry/index.json @@ -1,6 +1,6 @@ { "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 14747475..0ae24966 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c", - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, "compatibility": { "openai": "pass", @@ -48,10 +48,10 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" } diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index f1398131..75697bfe 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14T00:32:04Z", + "generated_at": "2026-06-14T00:52:22Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index b188c644..dfcdabf5 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 153, - "script_file_count": 94, + "python_file_count": 154, + "script_file_count": 95, "test_file_count": 59, - "internal_module_count": 19, + "internal_module_count": 20, "cli_script_count": 76, "command_handler_count": 34, "warn_line_threshold": 900, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 168317fb..8396d083 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,10 +5,10 @@ Generated at: `2026-06-13` ## Summary - decision: `pass` -- python files: `153` -- scripts: `94` +- python files: `154` +- scripts: `95` - tests: `59` -- internal modules: `19` +- internal modules: `20` - CLI scripts: `76` - Yao CLI command handlers: `34` - largest file lines: `899` diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 09f23f59..96ea6484 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,24 +3,24 @@ "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", - "commit": "bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126", + "commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308", "git_status": { "available": true, "dirty": true, - "changed_file_count": 27, + "changed_file_count": 50, "sample": [ " M AGENTS.md", - " M evidence/world_class/README.md", - " M evidence/world_class/intake.schema.json", " M registry/index.json", " M registry/packages/yao-meta-skill.json", " M reports/adoption_drift_report.json", + " M reports/adoption_drift_report.md", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", " M reports/benchmark_reproducibility.md", " M reports/context_budget.json", - " M reports/output_execution_runs.json", - " M reports/output_execution_runs.md", - " M reports/package_verification.json" + " M reports/install_simulation.json", + " M reports/install_simulation.md" ] }, "summary": { @@ -42,7 +42,7 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 27 + "changed_file_count": 50 }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -116,7 +116,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "9b77977fc455f5cc473317efb0a71ae07a13bb1ddc5874ba5acb296659c5bac0" + "sha256": "0ff38df2c18b25b79b9bf36e3055908a57d920ff45fa697f8281ce3a960f61b5" }, { "label": "blind_review", @@ -150,57 +150,57 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 97005, - "sha256": "dc188ff6756e94c207ebdd2e182dcb2432cf44d8f8f824c10ff31f6ba0a4078f" + "bytes": 97621, + "sha256": "3c95ea60b1c941d1adae1827ca0ec312140d72db4d45e512ace8636f75378ba2" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, - "bytes": 20367, - "sha256": "3698aad5388cad659840d1d31dbc9c694e0067b1de1e84423336a34e22df9625" + "bytes": 20499, + "sha256": "4edb5722c888f2fe5fc6c3141535ab5d550b6b1c05e9b4230b77e6712a982fb4" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "b9acd15b662da8835b0c2ba0ad554a409f3265c2c779753d68f8a8dc5fdd5820" + "sha256": "ad7900024ed6cadd5ea717ba22e38f38402e1a883bff119da24f7ecfd92c7dd8" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19325, - "sha256": "0e87470fb95411379ee79420db3c7069433d086378a69b78da1f4ad2bf45edd8" + "sha256": "dd12856ce8306600e089b44018ae4f7fae66366fba60f9408da885fbf5ab8035" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "97055f63c699b2812bab1a0f05460f1065a6211164b4c22c9bebc8aab6ef012f" + "bytes": 8758, + "sha256": "aabe72c3f5bde128ff74a24aef3e35f0ad449a431b9a932e92ec4ae5f3f308e4" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14309, - "sha256": "9c7b86742ed4dd5c930fbc3572d943937a4817bbd465d89da12f0929911607d0" + "sha256": "0c74deea906e75f6a116e2e3d043e134ce820bd12782af18428b3c04a02e6bd0" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 10083, - "sha256": "194d5a681920c3aca5e454d8839cdec551a076d54e360424831c5f312d1367b1" + "bytes": 19532, + "sha256": "a9bd392cd45f3accabe9b5befb9090456f383b1edbc323cab7d20930179dc229" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 11308, - "sha256": "dc4cf8cf18899bdf10d32fe72bb8f306ee297106e6cc25fd041d56e36631af63" + "bytes": 11613, + "sha256": "0bff14542475657a1261c1afcd87ffabc5d5c5e75bcd20afcfeacc44b16b7c14" }, { "label": "world_class_evidence_intake", @@ -214,14 +214,14 @@ "path": "reports/world_class_submission_review.json", "exists": true, "bytes": 7263, - "sha256": "185a8cab25e7155ebd80cb66049c0965e4dc2d7cc38991f17464946f87fd26f2" + "sha256": "4f03edb2ef28d9dfd27a393f8058d6e27d2b10fa8cbfe35f473e003556b9ffe0" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, "bytes": 15002, - "sha256": "3d643dc8170b50a37f91ffb190f69ad1e0d32f64914ebe2d731226728ac28cdc" + "sha256": "614742ea666c64cdb68e10fc77022c31e68abe82ed88325aea21acb2f277063e" }, { "label": "world_class_operator_runbook_markdown", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 89573749..9b13c349 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,7 +1,7 @@ # Benchmark Reproducibility Generated at: `2026-06-14` -Commit: `bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126` +Commit: `48f2826b8c9ebad3a9cc90e6166f4fd253f98308` Working tree dirty at generation: `true` ## Summary @@ -16,7 +16,7 @@ Working tree dirty at generation: `true` - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- changed files at generation: `27` +- changed files at generation: `50` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. @@ -40,22 +40,22 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `9b77977fc455` | +| output_execution | `reports/output_execution_runs.json` | present | `0ff38df2c18b` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `dc188ff6756e` | -| python_compatibility | `reports/python_compatibility.json` | present | `3698aad5388c` | -| registry_audit | `reports/registry_audit.json` | present | `b9acd15b662d` | -| package_verification | `reports/package_verification.json` | present | `0e87470fb954` | -| install_simulation | `reports/install_simulation.json` | present | `97055f63c699` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `9c7b86742ed4` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `194d5a681920` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `dc4cf8cf1889` | +| trust_report | `reports/security_trust_report.json` | present | `3c95ea60b1c9` | +| python_compatibility | `reports/python_compatibility.json` | present | `4edb5722c888` | +| registry_audit | `reports/registry_audit.json` | present | `ad7900024ed6` | +| package_verification | `reports/package_verification.json` | present | `dd12856ce830` | +| install_simulation | `reports/install_simulation.json` | present | `aabe72c3f5bd` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `0c74deea906e` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `a9bd392cd45f` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0bff14542475` | | world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `2cb7e6beb565` | -| world_class_submission_review | `reports/world_class_submission_review.json` | present | `185a8cab25e7` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `3d643dc8170b` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `4f03edb2ef28` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `614742ea666c` | | world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `f67ca3d0aa9c` | | world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `963026f551dc` | | world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `250d616b028c` | diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index dfb56a41..bf18763e 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, diff --git a/reports/context_budget.json b/reports/context_budget.json index d1672d78..8ac8fd13 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1178333, + "other_text_tokens": 1183524, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1179084, - "relevant_file_count": 498, + "estimated_total_text_tokens": 1184275, + "relevant_file_count": 500, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 291d2ad4..38051603 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -1,14 +1,14 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 573, + "archive_entry_count": 574, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -26,7 +26,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -272,9 +272,9 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } } diff --git a/reports/install_simulation.md b/reports/install_simulation.md index 6750e007..b0d16dae 100644 --- a/reports/install_simulation.md +++ b/reports/install_simulation.md @@ -1,7 +1,7 @@ # Install Simulation - OK: `True` -- Package directory: `dist` +- Package directory: `tests/tmp_review_studio/dist` - Archive extracted: `True` - Entrypoint loaded: `True` - Manifest loaded: `True` @@ -16,7 +16,7 @@ | Check | Status | Detail | | --- | --- | --- | -| `archive-present` | `pass` | Package archive exists: dist/yao-meta-skill.zip | +| `archive-present` | `pass` | Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip | | `archive-safe-paths` | `pass` | Archive has no absolute or parent-traversal entries | | `single-top-level` | `pass` | Archive top-level directory is yao-meta-skill | | `entrypoint-load` | `pass` | Installed SKILL.md frontmatter is readable | diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 863bb839..04d82c8d 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.72, + "duration_ms": 30.68, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.92, + "duration_ms": 28.53, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.18, + "duration_ms": 29.59, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.49, + "duration_ms": 31.84, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.68, + "duration_ms": 30.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.46, + "duration_ms": 27.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.04, + "duration_ms": 26.76, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.01, + "duration_ms": 26.87, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.99, + "duration_ms": 26.9, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.34, + "duration_ms": 27.3, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 6c6aeaf3..083d91b8 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 24.72 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 23.92 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 24.18 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 25.49 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 24.68 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 24.46 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 24.04 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 24.01 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 23.99 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 24.34 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 30.68 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 28.53 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 29.59 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.84 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 30.95 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.6 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.76 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.87 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.9 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.3 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index c071aab5..01c17c20 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", - "archive_entry_count": 573, + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", + "archive_entry_count": 574, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index b452b759..bc468a28 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e` +- Archive SHA256: `76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe` - Failures: `0` - Warnings: `0` diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 850f303b..0b4f6d3b 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 156, + "file_count": 157, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -562,6 +562,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_evidence_contract.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/yao.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index c9259ec3..cef6ad7d 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-13` - decision: `pass` - target python: `3.11` -- files scanned: `156` +- files scanned: `157` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 37ce36e5..91979125 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c", - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, "compatibility": { "openai": "pass", @@ -53,16 +53,16 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index f17044fd..777807a0 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c` -- Archive SHA256: `8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e` +- Package SHA256: `c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb` +- Archive SHA256: `76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 0f9bd02c..8d61ee49 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -434,12 +434,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

94 scripts scanned; secrets found

Py Compat0

156 files scanned for Python 3.11

Arch Debt0

899 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

73 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

573 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

95 scripts scanned; secrets found

Py Compat0

157 files scanned for Python 3.11

Arch Debt0

899 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

73 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

574 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 94 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 156 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

153 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 95 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 157 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

154 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -506,12 +506,12 @@
-

信任报告

Secret
0
脚本数
94
网络脚本
3
Help 失败
0
包体哈希
08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c
+

信任报告

Secret
0
脚本数
95
网络脚本
3
Help 失败
0
包体哈希
c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

-

Python 兼容

目标 Python
3.11
文件数
156
问题数
0
语法错误
0
F-string 3.11
0
+

Python 兼容

目标 Python
3.11
文件数
157
问题数
0
语法错误
0
F-string 3.11
0

解释器边界

CI 和发布审查以 Python 3.11 兼容为底线;本地更高版本允许的新语法不能绕过兼容门禁。

@@ -574,12 +574,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
573
失败数
0
警告数
0
归档哈希
8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e
+

包体验证

目标数
4
Adapter
4
归档存在
Zip 条目
574
失败数
0
警告数
0
归档哈希
76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe
diff --git a/reports/review-studio.json b/reports/review-studio.json index 9aa86fd3..9d0e3fd3 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 94 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 95 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -67,7 +67,7 @@ "key": "python-compat", "label": "Python 兼容", "status": "pass", - "detail": "Python 3.11; 156 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", + "detail": "Python 3.11; 157 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards", "evidence": "reports/python_compatibility.json", "link": "python_compatibility.md" }, @@ -75,7 +75,7 @@ "key": "architecture-maintainability", "label": "架构维护", "status": "pass", - "detail": "153 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers", + "detail": "154 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers", "evidence": "reports/architecture_maintainability.json", "link": "architecture_maintainability.md" }, @@ -851,7 +851,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 94 + "file_count": 95 }, { "path": "evals", @@ -866,7 +866,7 @@ "file_count": 203 } ], - "file_count": 361, + "file_count": 362, "folder_count": 4, "distribution": [ { @@ -891,7 +891,7 @@ }, { "label": "scripts", - "value": 94 + "value": 95 }, { "label": "evals", @@ -1020,7 +1020,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 94 + "file_count": 95 }, { "path": "evals", @@ -1349,9 +1349,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 45 + "changed_file_count": 50 }, - "commit": "bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126", + "commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1434,9 +1434,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 178, - "script_count": 94, - "internal_module_count": 17, + "scanned_files": 179, + "script_count": 95, + "internal_module_count": 18, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1454,8 +1454,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 178, - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "package_hash_file_count": 179, + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" }, "skill_atlas": { "skill_count": 12, @@ -1493,8 +1493,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c", - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, "compatibility": { "openai": "pass", @@ -1525,12 +1525,12 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1541,8 +1541,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", - "archive_entry_count": 573, + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", + "archive_entry_count": 574, "failure_count": 0, "warning_count": 0 }, @@ -1553,7 +1553,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 573, + "archive_entry_count": 574, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1620,12 +1620,12 @@ { "field": "archive_sha256", "from": "", - "to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" } ] }, @@ -1745,7 +1745,10 @@ "summary": { "audit_decision": "continue-iteration", "world_class_ready": false, + "audit_world_class_ready": false, "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, "task_count": 4, "human_task_count": 1, "external_task_count": 3, @@ -1920,6 +1923,7 @@ "ok": true, "summary": { "ledger_entry_count": 4, + "source_accepted_count": 0, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, @@ -1928,6 +1932,7 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -1940,6 +1945,7 @@ "owner": "operator with provider credentials", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "provenance_requirements": [ @@ -1995,6 +2001,7 @@ "owner": "human reviewer", "status": "pending", "source_status": "human_required", + "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "provenance_requirements": [ @@ -2054,6 +2061,7 @@ "owner": "target client or installer integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", "provenance_requirements": [ @@ -2116,6 +2124,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ @@ -2169,7 +2178,8 @@ "source_plan": { "json": "reports/world_class_evidence_plan.json", "markdown": "reports/world_class_evidence_plan.md", - "task_count": 4 + "task_count": 4, + "evidence_requirement_count": 4 } }, "synthesis_highlights": [ @@ -3794,7 +3804,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.72, + "duration_ms": 28.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3822,7 +3832,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.92, + "duration_ms": 27.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3845,7 +3855,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.18, + "duration_ms": 27.7, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3873,7 +3883,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.49, + "duration_ms": 27.35, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3896,7 +3906,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.68, + "duration_ms": 28.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3924,7 +3934,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.46, + "duration_ms": 28.65, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3947,7 +3957,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.04, + "duration_ms": 28.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3974,7 +3984,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.01, + "duration_ms": 27.75, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3997,7 +4007,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.99, + "duration_ms": 27.86, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4026,7 +4036,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 24.34, + "duration_ms": 27.32, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4866,7 +4876,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "94 scripts; secrets 0; help failures 0", + "current": "95 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -4940,7 +4950,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 573; install failures 0", + "current": "archive entries 574; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -5361,7 +5371,7 @@ "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, @@ -10221,9 +10231,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 178, - "script_count": 94, - "internal_module_count": 17, + "scanned_files": 179, + "script_count": 95, + "internal_module_count": 18, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -10241,8 +10251,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 178, - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "package_hash_file_count": 179, + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" }, "failures": [], "warnings": [], @@ -11483,6 +11493,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_evidence_contract.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to share intake validation and artifact integrity checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao.py", "interface": "cli", @@ -11616,7 +11640,7 @@ "checked_count": 77, "passed_count": 77, "failed_count": 0, - "skipped_count": 17, + "skipped_count": 18, "failed_scripts": [], "results": [ { @@ -12435,6 +12459,10 @@ "path": "scripts/skill_report_model.py", "reason": "internal module" }, + { + "path": "scripts/world_class_evidence_contract.py", + "reason": "internal module" + }, { "path": "scripts/yao_cli_config.py", "reason": "internal module" @@ -12644,7 +12672,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 156, + "file_count": 157, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -13201,6 +13229,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_evidence_contract.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/yao.py", "ok": true, @@ -13610,10 +13644,10 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 153, - "script_file_count": 94, + "python_file_count": 154, + "script_file_count": 95, "test_file_count": 59, - "internal_module_count": 19, + "internal_module_count": 20, "cli_script_count": 76, "command_handler_count": 34, "warn_line_threshold": 900, @@ -13724,10 +13758,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1178333, + "other_text_tokens": 1183524, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1179084, - "relevant_file_count": 498, + "estimated_total_text_tokens": 1184275, + "relevant_file_count": 500, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -14678,6 +14712,7 @@ "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", "scripts/yao.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py", @@ -15580,7 +15615,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14T00:32:04Z", + "generated_at": "2026-06-14T00:50:30Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -15768,10 +15803,11 @@ "world_class_evidence_ledger": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "ledger_entry_count": 4, + "source_accepted_count": 0, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, @@ -15780,6 +15816,7 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -15792,6 +15829,7 @@ "owner": "operator with provider credentials", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "provenance_requirements": [ @@ -15847,6 +15885,7 @@ "owner": "human reviewer", "status": "pending", "source_status": "human_required", + "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "provenance_requirements": [ @@ -15906,6 +15945,7 @@ "owner": "target client or installer integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", "provenance_requirements": [ @@ -15968,6 +16008,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ @@ -16021,11 +16062,13 @@ "source_plan": { "json": "reports/world_class_evidence_plan.json", "markdown": "reports/world_class_evidence_plan.md", - "task_count": 4 + "task_count": 4, + "evidence_requirement_count": 4 }, "submissions": { "directory": "evidence/world_class/submissions", - "ledger_counts_submission_as_completion": false + "ledger_counts_submission_as_completion": false, + "source_pass_requires_valid_submission": true }, "artifacts": { "json": "reports/world_class_evidence_ledger.json", @@ -16036,7 +16079,7 @@ "world_class_evidence_intake": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "schema_present": true, @@ -16330,7 +16373,7 @@ "world_class_submission_review": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "review_item_count": 4, @@ -16506,7 +16549,7 @@ "world_class_operator_runbook": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "evidence_item_count": 4, @@ -16825,7 +16868,7 @@ "world_class_claim_guard": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "ledger_ready_to_claim_world_class": false, @@ -17189,8 +17232,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c", - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, "compatibility": { "openai": "pass", @@ -17221,16 +17264,16 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "package_count": 1, "packages": [ { @@ -17246,7 +17289,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" } ] }, @@ -17269,8 +17312,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", - "archive_entry_count": 573, + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", + "archive_entry_count": 574, "failure_count": 0, "warning_count": 0 }, @@ -17928,14 +17971,14 @@ "install_simulation": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 573, + "archive_entry_count": 574, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -17953,7 +17996,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -18199,16 +18242,16 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } }, "upgrade_check": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -18277,12 +18320,12 @@ { "field": "archive_sha256", "from": "", - "to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" } ] }, @@ -18297,8 +18340,8 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } }, "manifest": { diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 0f475ac5..bcabbe82 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 178, - "script_count": 94, - "internal_module_count": 17, + "scanned_files": 179, + "script_count": 95, + "internal_module_count": 18, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 178, - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "package_hash_file_count": 179, + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" }, "failures": [], "warnings": [], @@ -1264,6 +1264,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_evidence_contract.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to share intake validation and artifact integrity checks.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao.py", "interface": "cli", @@ -1397,7 +1411,7 @@ "checked_count": 77, "passed_count": 77, "failed_count": 0, - "skipped_count": 17, + "skipped_count": 18, "failed_scripts": [], "results": [ { @@ -2216,6 +2230,10 @@ "path": "scripts/skill_report_model.py", "reason": "internal module" }, + { + "path": "scripts/world_class_evidence_contract.py", + "reason": "internal module" + }, { "path": "scripts/yao_cli_config.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 4033f209..47ec8a56 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `178` -- Scripts: `94` -- Internal script modules: `17` +- Scanned files: `179` +- Scripts: `95` +- Internal script modules: `18` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `178` -- Package SHA256: `08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c` +- Package hash files: `179` +- Package SHA256: `c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb` ## Failures @@ -149,6 +149,7 @@ | scripts/upgrade_check.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/validate_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | | scripts/yao.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/yao_cli_config.py | internal-module | True | False | False | False | False | False | False | Imported by yao.py for CLI target maps and side-effect-free shaping helpers. | | scripts/yao_cli_create_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep skill creation and quickstart command handlers out of the CLI orchestrator. | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index edee53be..68891621 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布361项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布362项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index fd317a77..2ea2e2f2 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -437,7 +437,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 94 + "file_count": 95 }, { "path": "evals", @@ -452,7 +452,7 @@ "file_count": 203 } ], - "file_count": 361, + "file_count": 362, "folder_count": 4, "distribution": [ { @@ -477,7 +477,7 @@ }, { "label": "scripts", - "value": 94 + "value": 95 }, { "label": "evals", @@ -606,7 +606,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 94 + "file_count": 95 }, { "path": "evals", @@ -935,9 +935,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 45 + "changed_file_count": 50 }, - "commit": "bd7f1d8cccbebac1222c0ee7bbc01c1e3b991126", + "commit": "48f2826b8c9ebad3a9cc90e6166f4fd253f98308", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1020,9 +1020,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 178, - "script_count": 94, - "internal_module_count": 17, + "scanned_files": 179, + "script_count": 95, + "internal_module_count": 18, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1040,8 +1040,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 178, - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "package_hash_file_count": 179, + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" }, "skill_atlas": { "skill_count": 12, @@ -1079,8 +1079,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c", - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "package_sha256": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, "compatibility": { "openai": "pass", @@ -1111,12 +1111,12 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-14" + "generated_at": "2026-06-13" }, "failures": [], "warnings": [] @@ -1127,8 +1127,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e", - "archive_entry_count": 573, + "archive_sha256": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe", + "archive_entry_count": 574, "failure_count": 0, "warning_count": 0 }, @@ -1139,7 +1139,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 573, + "archive_entry_count": 574, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1206,12 +1206,12 @@ { "field": "archive_sha256", "from": "", - "to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" } ] }, @@ -1331,7 +1331,10 @@ "summary": { "audit_decision": "continue-iteration", "world_class_ready": false, + "audit_world_class_ready": false, "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, "task_count": 4, "human_task_count": 1, "external_task_count": 3, @@ -1506,6 +1509,7 @@ "ok": true, "summary": { "ledger_entry_count": 4, + "source_accepted_count": 0, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, @@ -1514,6 +1518,7 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -1526,6 +1531,7 @@ "owner": "operator with provider credentials", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "provenance_requirements": [ @@ -1581,6 +1587,7 @@ "owner": "human reviewer", "status": "pending", "source_status": "human_required", + "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "provenance_requirements": [ @@ -1640,6 +1647,7 @@ "owner": "target client or installer integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", "provenance_requirements": [ @@ -1702,6 +1710,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ @@ -1755,7 +1764,8 @@ "source_plan": { "json": "reports/world_class_evidence_plan.json", "markdown": "reports/world_class_evidence_plan.md", - "task_count": 4 + "task_count": 4, + "evidence_requirement_count": 4 } }, "synthesis_highlights": [ diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 09f7c547..d7f86aec 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -151,6 +151,7 @@ "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", "scripts/yao.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index d085c79e..a96090b4 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 94; help failures 0", + "current": "secrets 0; scripts 95; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 573; install failures 0; permission failures 0", + "current": "zip entries 574; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 1bed703e..b385bb16 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,11 +23,11 @@ Generated at: `2026-06-13` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 24; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 94; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 95; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 573; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 574; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 7f768f25..45e97363 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -122,7 +122,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "94 scripts; secrets 0; help failures 0", + "current": "95 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -196,7 +196,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 573; install failures 0", + "current": "archive entries 574; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index d60139bc..2a7ef08c 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -21,9 +21,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 94 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 95 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 573; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 574; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index e40544d3..4b80c227 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "8d1c803b4008860c73b47f8abeed721877304521c2e2da11083aeb2e66ce156e" + "to": "76793bd367e9aaf6c25e6b11ac1139156950827bd87ba3dd8a1bdc883f7d70fe" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "08d744e0aa33d726b497ecdbdb6eaf2a5ec666a6ea3938068f6007be27f5021c" + "to": "c42643ea854dd90bb03ca607595f6a6b2c00118da64a78e8bdbb4392d7f204fb" } ] }, @@ -90,7 +90,7 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } } diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index 88dc21e5..edf4d040 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "ledger_ready_to_claim_world_class": false, diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index b3eaa58f..3318385b 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -1,6 +1,6 @@ # World-Class Claim Guard -Generated at: `2026-06-14` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index 2cfe6e1e..356c6383 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "schema_present": true, diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index 04ebda85..4376838f 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -1,6 +1,6 @@ # World-Class Evidence Intake -Generated at: `2026-06-14` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index efea089e..3225daee 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -1,10 +1,11 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "ledger_entry_count": 4, + "source_accepted_count": 0, "accepted_count": 0, "pending_count": 4, "human_pending_count": 1, @@ -13,6 +14,7 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "submitted_but_pending_count": 0, + "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, "ready_to_claim_world_class": false, "decision": "evidence-pending" @@ -25,6 +27,7 @@ "owner": "operator with provider credentials", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "model-executed 0; token-observed 0", "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "provenance_requirements": [ @@ -80,6 +83,7 @@ "owner": "human reviewer", "status": "pending", "source_status": "human_required", + "source_accepted": false, "current": "0/5 decisions; pending 5", "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", "provenance_requirements": [ @@ -139,6 +143,7 @@ "owner": "target client or installer integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "native-enforced targets 0; installer-enforced targets 4", "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", "provenance_requirements": [ @@ -201,6 +206,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", + "source_accepted": false, "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ @@ -254,11 +260,13 @@ "source_plan": { "json": "reports/world_class_evidence_plan.json", "markdown": "reports/world_class_evidence_plan.md", - "task_count": 4 + "task_count": 4, + "evidence_requirement_count": 4 }, "submissions": { "directory": "evidence/world_class/submissions", - "ledger_counts_submission_as_completion": false + "ledger_counts_submission_as_completion": false, + "source_pass_requires_valid_submission": true }, "artifacts": { "json": "reports/world_class_evidence_ledger.json", diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index e6b0f882..71d63aef 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -1,22 +1,24 @@ # World-Class Evidence Ledger -Generated at: `2026-06-14` +Generated at: `2026-06-13` ## Summary - decision: `evidence-pending` - ready to claim world-class: `false` - entries: `4` +- source accepted: `0` - accepted: `0` - pending: `4` - human pending: `1` - external pending: `3` - submitted entries: `0` - submitted but pending: `0` +- source accepted without valid submission: `0` - invalid submissions: `0` - overclaim guard active: `true` -This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence. +This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence. ## Ledger diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index ae80126b..47ea5508 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -1,12 +1,15 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "audit_decision": "continue-iteration", "world_class_ready": false, + "audit_world_class_ready": false, "ready_to_claim_world_class": false, + "ledger_completion_required": true, + "evidence_requirement_count": 4, "task_count": 4, "human_task_count": 1, "external_task_count": 3, @@ -171,6 +174,164 @@ "audit_next_action": "Install a real client against the native host and import production metadata-only events." } ], + "evidence_requirements": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "status": "external_required", + "category": "external", + "owner": "operator with provider credentials", + "current": "model-executed 0; token-observed 0", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "runbook": [ + "YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY= python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake ." + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/provider-holdout.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "audit_next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "status": "human_required", + "category": "human", + "owner": "human reviewer", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "runbook": [ + "python3 scripts/yao.py output-review-kit --write-template", + "Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.", + "python3 scripts/adjudicate_output_review.py --write-template", + "Edit reports/output_review_decisions.json with winner_variant values and reviewer metadata.", + "python3 scripts/yao.py output-review", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake ." + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_kit.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/human-adjudication.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ], + "audit_next_action": "Record real A/B choices in the decision template, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "status": "external_required", + "category": "external", + "owner": "target client or installer integrator", + "current": "native-enforced targets 0; installer-enforced targets 4", + "objective": "Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.", + "runbook": [ + "Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.", + "Update the generated target adapter only when the guard is actually enforced by that target.", + "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip", + "python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation", + "python3 scripts/yao.py runtime-permissions . --package-dir dist", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake ." + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "reports/install_simulation.json", + "reports/install_simulation.md", + "security/permission_policy.json", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-permission-enforcement.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "audit_next_action": "Integrate a real target-client or external installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "status": "external_required", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "current": "external source events 0; adoption samples 1", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "runbook": [ + "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", + "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", + "python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl", + "python3 scripts/yao.py skill-atlas --workspace-root .", + "python3 scripts/yao.py skill-os2-audit . --generated-at ", + "Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.", + "python3 scripts/yao.py world-class-intake ." + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py", + "evidence/world_class/intake.schema.json", + "evidence/world_class/templates/native-client-telemetry.intake.json", + "reports/world_class_evidence_intake.json", + "reports/world_class_evidence_intake.md" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "audit_next_action": "Install a real client against the native host and import production metadata-only events." + } + ], "source_audit": { "json": "reports/skill_os2_audit.json", "markdown": "reports/skill_os2_audit.md", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index c8f1d162..63bf0848 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -1,17 +1,19 @@ # World-Class Evidence Plan -Generated at: `2026-06-14` +Generated at: `2026-06-13` ## Summary - decision: `collect-external-evidence` - audit decision: `continue-iteration` - ready to claim world-class: `false` +- ledger completion required: `true` +- evidence requirements: `4` - tasks: `4` - human tasks: `1` - external tasks: `3` -This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan as completion. +This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan or source-report pass as completion; the ledger must still validate accepted submissions. ## Task Table diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index c7f73901..c7a608a0 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "evidence_item_count": 4, diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index 449a0dc2..5cc913bf 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -1,6 +1,6 @@ # World-Class Operator Runbook -Generated at: `2026-06-14` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index b462c0bb..f5cd09b6 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "review_item_count": 4, diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index f8f7527b..11389553 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -1,6 +1,6 @@ # World-Class Submission Review -Generated at: `2026-06-14` +Generated at: `2026-06-13` ## Summary diff --git a/scripts/render_world_class_evidence_intake.py b/scripts/render_world_class_evidence_intake.py index 50000742..576387fb 100644 --- a/scripts/render_world_class_evidence_intake.py +++ b/scripts/render_world_class_evidence_intake.py @@ -1,255 +1,19 @@ #!/usr/bin/env python3 import argparse -import hashlib import json -import re import shlex from datetime import date from pathlib import Path from typing import Any from render_world_class_evidence_ledger import build_ledger +from world_class_evidence_contract import load_json, rel_path, validate_payload ROOT = Path(__file__).resolve().parent.parent SCRIPT_INTERFACE = "cli" SCRIPT_INTERFACE_REASON = "Validates world-class human and external evidence intake packets before ledger review." -REQUIRED_TOP_LEVEL = [ - "schema_version", - "evidence_key", - "template_only", - "category", - "source_type", - "submitted_by", - "submitted_at", - "summary", - "artifact_refs", - "provenance", - "privacy", - "anti_overclaim", - "attestation", -] -REQUIRED_PRIVACY_FALSE = [ - "raw_user_content_included", - "raw_provider_prompt_included", - "credentials_included", - "secrets_included", -] -REQUIRED_ANTI_OVERCLAIM_FALSE = [ - "planned_work_counts_as_evidence", - "metadata_fallback_counts_as_native_enforcement", - "pending_review_counts_as_human_decision", - "local_command_runner_counts_as_provider_model", -] -REQUIRED_ATTESTATION_TRUE = [ - "real_external_or_human_evidence", - "reviewer_or_operator_identity_present", - "artifact_refs_reviewed", - "privacy_contract_satisfied", -] -EXPECTED_SOURCE_TYPES = { - "provider-holdout": "provider-output-eval", - "human-adjudication": "blind-ab-review", - "native-permission-enforcement": "runtime-permission-guard", - "native-client-telemetry": "native-client-telemetry", -} -SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") -DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"} - - -def load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {} - return payload if isinstance(payload, dict) else {} - - -def rel_path(path: Path, root: Path) -> str: - try: - return str(path.resolve().relative_to(root.resolve())) - except ValueError: - return str(path.resolve()) - - -def add_error(errors: list[str], condition: bool, message: str) -> None: - if not condition: - errors.append(message) - - -def sha256_file(path: Path) -> str: - digest = hashlib.sha256() - with path.open("rb") as handle: - for chunk in iter(lambda: handle.read(1024 * 1024), b""): - digest.update(chunk) - return digest.hexdigest() - - -def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]: - text = raw_path.strip() - if any(token in text for token in ("<", ">", "*", "?")): - return None, "artifact_refs path must be concrete, not a placeholder or glob" - candidate = Path(text) - if candidate.is_absolute(): - return None, "artifact_refs path must be relative to the skill directory" - if ".." in candidate.parts: - return None, "artifact_refs path must not escape the skill directory" - resolved = (root / candidate).resolve() - try: - resolved.relative_to(root.resolve()) - except ValueError: - return None, "artifact_refs path must stay inside the skill directory" - return resolved, None - - -def validate_artifact_refs( - payload: dict[str, Any], - errors: list[str], - *, - root: Path, - template_expected: bool, -) -> dict[str, int]: - refs = payload.get("artifact_refs") - add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference") - stats = { - "artifact_ref_count": len(refs) if isinstance(refs, list) else 0, - "artifact_existing_count": 0, - "artifact_sha256_verified_count": 0, - } - if not isinstance(refs, list): - return stats - for index, ref in enumerate(refs): - if not isinstance(ref, dict): - errors.append(f"artifact_refs[{index}] must be an object") - continue - path_text = str(ref.get("path", "")).strip() - add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required") - add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required") - add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content") - if template_expected or not path_text: - continue - resolved, path_error = resolve_artifact_path(path_text, root) - if path_error: - errors.append(f"artifact_refs[{index}].path {path_error}") - continue - rel = rel_path(resolved, root) - if rel in DISALLOWED_REAL_ARTIFACTS: - errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs") - if not resolved.exists() or not resolved.is_file(): - errors.append(f"artifact_refs[{index}].path does not exist as a local file") - continue - stats["artifact_existing_count"] += 1 - declared = str(ref.get("sha256", "")).strip() - if not declared: - errors.append(f"artifact_refs[{index}].sha256 is required for a real submission") - continue - if not SHA256_RE.match(declared): - errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest") - continue - actual = sha256_file(resolved) - if actual.lower() != declared.lower(): - errors.append(f"artifact_refs[{index}].sha256 does not match local artifact") - continue - stats["artifact_sha256_verified_count"] += 1 - return stats - - -def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None: - key = str(payload.get("evidence_key", "")) - provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} - if key == "provider-holdout": - add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required") - add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required") - add_error( - errors, - provenance.get("credential_material_committed") is False, - "provider-holdout must attest credential_material_committed is false", - ) - elif key == "human-adjudication": - add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required") - add_error( - errors, - provenance.get("answer_key_opened_after_decisions") is True, - "human-adjudication must attest answer_key_opened_after_decisions is true", - ) - elif key == "native-permission-enforcement": - add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required") - add_error( - errors, - bool(str(provenance.get("guard_location", "")).strip()), - "native-permission-enforcement provenance.guard_location is required", - ) - add_error( - errors, - str(provenance.get("guard_scope", "")).strip() - in {"target-client-native", "external-installer-runtime-guard"}, - "native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard", - ) - add_error( - errors, - provenance.get("guard_blocks_undeclared_capability") is True, - "native-permission-enforcement must attest guard_blocks_undeclared_capability is true", - ) - add_error( - errors, - provenance.get("metadata_fallback_retained_for_other_targets") is True, - "native-permission-enforcement must retain metadata fallback for non-native targets", - ) - elif key == "native-client-telemetry": - add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required") - add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external") - add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only") - - -def validate_payload( - payload: dict[str, Any], - entry: dict[str, Any], - *, - path: Path, - root: Path, - template_expected: bool, -) -> dict[str, Any]: - errors: list[str] = [] - for key in REQUIRED_TOP_LEVEL: - add_error(errors, key in payload, f"missing {key}") - evidence_key = str(entry.get("key", "")) - add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0") - add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}") - add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}") - add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}") - add_error( - errors, - payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key), - f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}", - ) - add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required") - add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required") - add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required") - artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected) - privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {} - for key in REQUIRED_PRIVACY_FALSE: - add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false") - anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {} - for key in REQUIRED_ANTI_OVERCLAIM_FALSE: - add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false") - validate_evidence_specific(payload, errors) - attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {} - if not template_expected: - for key in REQUIRED_ATTESTATION_TRUE: - add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission") - return { - "path": rel_path(path, root), - "evidence_key": evidence_key, - "status": "pass" if not errors else "fail", - "template_only": template_expected, - "artifact_integrity": artifact_integrity, - "errors": errors, - } - - def template_paths(skill_dir: Path, keys: list[str]) -> dict[str, Path]: template_dir = skill_dir / "evidence" / "world_class" / "templates" return {key: template_dir / f"{key}.intake.json" for key in keys} diff --git a/scripts/render_world_class_evidence_ledger.py b/scripts/render_world_class_evidence_ledger.py index 68ec50ea..38a49e40 100644 --- a/scripts/render_world_class_evidence_ledger.py +++ b/scripts/render_world_class_evidence_ledger.py @@ -6,6 +6,7 @@ from pathlib import Path from typing import Any from render_world_class_evidence_plan import build_plan +from world_class_evidence_contract import load_json, load_json_with_status, rel_path, validate_payload ROOT = Path(__file__).resolve().parent.parent @@ -13,35 +14,6 @@ SCRIPT_INTERFACE = "cli" SCRIPT_INTERFACE_REASON = "Renders a machine-checkable ledger for world-class external and human evidence gaps." -def load_json(path: Path) -> dict[str, Any]: - if not path.exists(): - return {} - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {} - return payload if isinstance(payload, dict) else {} - - -def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]: - if not path.exists(): - return {}, "missing" - try: - payload = json.loads(path.read_text(encoding="utf-8")) - except json.JSONDecodeError: - return {}, "invalid-json" - if not isinstance(payload, dict): - return {}, "invalid-json" - return payload, "present" - - -def rel_path(path: Path, root: Path) -> str: - try: - return str(path.resolve().relative_to(root.resolve())) - except ValueError: - return str(path.resolve()) - - def provider_state(skill_dir: Path) -> dict[str, Any]: summary = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {}) return { @@ -120,7 +92,8 @@ PROVENANCE_REQUIREMENTS = { } -def submission_state(skill_dir: Path, key: str, submissions_dir: Path) -> dict[str, Any]: +def submission_state(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]: + key = str(task.get("key", "")) path = submissions_dir / f"{key}.json" payload, load_status = load_json_with_status(path) if load_status != "present": @@ -132,32 +105,32 @@ def submission_state(skill_dir: Path, key: str, submissions_dir: Path) -> dict[s "privacy_contract_satisfied": False, "ledger_counts_as_completion": False, } - errors = [] - if payload.get("evidence_key") != key: - errors.append("evidence_key mismatch") - if payload.get("template_only") is not False: - errors.append("template_only must be false") + validation = validate_payload(payload, task, path=path, root=skill_dir, template_expected=False) refs = payload.get("artifact_refs", []) attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {} - status = "submitted" if not errors else "invalid-contract" + status = "submitted" if validation["status"] == "pass" else "invalid-contract" + artifact_integrity = validation.get("artifact_integrity", {}) return { "status": status, "path": rel_path(path, skill_dir), "submitted_by": payload.get("submitted_by", ""), "submitted_at": payload.get("submitted_at", ""), "artifact_ref_count": len(refs) if isinstance(refs, list) else 0, + "artifact_existing_count": artifact_integrity.get("artifact_existing_count", 0), + "artifact_sha256_verified_count": artifact_integrity.get("artifact_sha256_verified_count", 0), "attested_real_evidence": attestation.get("real_external_or_human_evidence") is True, "reviewer_or_operator_identity_present": attestation.get("reviewer_or_operator_identity_present") is True, "privacy_contract_satisfied": attestation.get("privacy_contract_satisfied") is True, - "errors": errors, + "errors": validation.get("errors", []), "ledger_counts_as_completion": False, } def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> dict[str, Any]: state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir) - accepted = bool(state.get("accepted")) - submission = submission_state(skill_dir, task["key"], submissions_dir) + submission = submission_state(skill_dir, task, submissions_dir) + source_accepted = bool(state.get("accepted")) + accepted = source_accepted and submission.get("status") == "submitted" return { "key": task["key"], "label": task["label"], @@ -165,6 +138,7 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> "owner": task["owner"], "status": "accepted" if accepted else "pending", "source_status": task["status"], + "source_accepted": source_accepted, "current": task["current"], "objective": task["objective"], "provenance_requirements": PROVENANCE_REQUIREMENTS.get(task["key"], ["release reviewer evidence"]), @@ -186,7 +160,9 @@ def build_entry(skill_dir: Path, task: dict[str, Any], submissions_dir: Path) -> def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | None = None) -> dict[str, Any]: plan = build_plan(skill_dir, generated_at) submissions_dir = submissions_dir or (skill_dir / "evidence" / "world_class" / "submissions") - entries = [build_entry(skill_dir, task, submissions_dir) for task in plan["tasks"]] + evidence_requirements = plan.get("evidence_requirements") or plan.get("tasks", []) + entries = [build_entry(skill_dir, task, submissions_dir) for task in evidence_requirements] + source_accepted_count = sum(1 for entry in entries if entry.get("source_accepted") is True) accepted_count = sum(1 for entry in entries if entry["status"] == "accepted") pending_count = len(entries) - accepted_count human_pending_count = sum(1 for entry in entries if entry["category"] == "human" and entry["status"] == "pending") @@ -197,7 +173,12 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non submitted_but_pending_count = sum( 1 for entry in entries if entry["submission_state"]["status"] == "submitted" and entry["status"] == "pending" ) - ready = pending_count == 0 and plan["summary"].get("world_class_ready") is True + source_accepted_without_valid_submission_count = sum( + 1 + for entry in entries + if entry.get("source_accepted") is True and entry["submission_state"]["status"] != "submitted" + ) + ready = bool(entries) and pending_count == 0 and plan["summary"].get("world_class_ready") is True return { "schema_version": "1.0", "ok": True, @@ -205,6 +186,7 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non "skill_dir": rel_path(skill_dir, ROOT), "summary": { "ledger_entry_count": len(entries), + "source_accepted_count": source_accepted_count, "accepted_count": accepted_count, "pending_count": pending_count, "human_pending_count": human_pending_count, @@ -213,6 +195,7 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non "missing_submission_count": missing_submission_count, "invalid_submission_count": invalid_submission_count, "submitted_but_pending_count": submitted_but_pending_count, + "source_accepted_without_valid_submission_count": source_accepted_without_valid_submission_count, "overclaim_guard_active": True, "ready_to_claim_world_class": ready, "decision": "ready-for-completion-audit" if ready else "evidence-pending", @@ -222,10 +205,12 @@ def build_ledger(skill_dir: Path, generated_at: str, submissions_dir: Path | Non "json": "reports/world_class_evidence_plan.json", "markdown": "reports/world_class_evidence_plan.md", "task_count": plan["summary"].get("task_count", 0), + "evidence_requirement_count": len(evidence_requirements), }, "submissions": { "directory": rel_path(submissions_dir, skill_dir), "ledger_counts_submission_as_completion": False, + "source_pass_requires_valid_submission": True, }, "artifacts": { "json": "reports/world_class_evidence_ledger.json", @@ -247,16 +232,18 @@ def render_markdown(ledger: dict[str, Any]) -> str: f"- decision: `{summary['decision']}`", f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", f"- entries: `{summary['ledger_entry_count']}`", + f"- source accepted: `{summary.get('source_accepted_count', 0)}`", f"- accepted: `{summary['accepted_count']}`", f"- pending: `{summary['pending_count']}`", f"- human pending: `{summary['human_pending_count']}`", f"- external pending: `{summary['external_pending_count']}`", f"- submitted entries: `{summary['submitted_entry_count']}`", f"- submitted but pending: `{summary['submitted_but_pending_count']}`", + f"- source accepted without valid submission: `{summary.get('source_accepted_without_valid_submission_count', 0)}`", f"- invalid submissions: `{summary['invalid_submission_count']}`", f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`", "", - "This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.", + "This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.", "", "## Ledger", "", diff --git a/scripts/render_world_class_evidence_plan.py b/scripts/render_world_class_evidence_plan.py index 3579bce2..31c097eb 100644 --- a/scripts/render_world_class_evidence_plan.py +++ b/scripts/render_world_class_evidence_plan.py @@ -175,11 +175,14 @@ def build_task(item: dict[str, Any]) -> dict[str, Any]: def build_plan(skill_dir: Path, generated_at: str) -> dict[str, Any]: audit = build_audit(skill_dir, generated_at) - tasks = [build_task(item) for item in audit["items"] if item["status"] != "pass"] + evidence_keys = set(TASK_TEMPLATES) + evidence_requirements = [build_task(item) for item in audit["items"] if item["key"] in evidence_keys] + tasks = [task for task in evidence_requirements if task["status"] != "pass"] + tasks.extend(build_task(item) for item in audit["items"] if item["status"] != "pass" and item["key"] not in evidence_keys) category_counts: dict[str, int] = {} for task in tasks: category_counts[task["category"]] = category_counts.get(task["category"], 0) + 1 - ready = len(tasks) == 0 and audit["summary"].get("world_class_ready") is True + audit_world_class_ready = audit["summary"].get("world_class_ready") is True return { "schema_version": "1.0", "ok": audit["ok"], @@ -188,14 +191,18 @@ def build_plan(skill_dir: Path, generated_at: str) -> dict[str, Any]: "summary": { "audit_decision": audit["summary"]["decision"], "world_class_ready": bool(audit["summary"]["world_class_ready"]), - "ready_to_claim_world_class": ready, + "audit_world_class_ready": audit_world_class_ready, + "ready_to_claim_world_class": False, + "ledger_completion_required": True, + "evidence_requirement_count": len(evidence_requirements), "task_count": len(tasks), "human_task_count": category_counts.get("human", 0), "external_task_count": category_counts.get("external", 0), "review_task_count": category_counts.get("review", 0), - "decision": "ready-for-completion-audit" if ready else "collect-external-evidence", + "decision": "audit-ready-ledger-required" if audit_world_class_ready and not tasks else "collect-external-evidence", }, "tasks": tasks, + "evidence_requirements": evidence_requirements, "source_audit": { "json": "reports/skill_os2_audit.json", "markdown": "reports/skill_os2_audit.md", @@ -222,11 +229,13 @@ def render_markdown(plan: dict[str, Any]) -> str: f"- decision: `{summary['decision']}`", f"- audit decision: `{summary['audit_decision']}`", f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- ledger completion required: `{str(summary.get('ledger_completion_required', True)).lower()}`", + f"- evidence requirements: `{summary.get('evidence_requirement_count', 0)}`", f"- tasks: `{summary['task_count']}`", f"- human tasks: `{summary['human_task_count']}`", f"- external tasks: `{summary['external_task_count']}`", "", - "This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan as completion.", + "This report is an execution plan for the remaining world-class evidence gaps. It does not count a plan or source-report pass as completion; the ledger must still validate accepted submissions.", "", "## Task Table", "", @@ -239,7 +248,7 @@ def render_markdown(plan: dict[str, Any]) -> str: f"| `{task['key']}` | `{task['status']}` | `{task['category']}` | {task['owner']} | {current} |" ) if not plan["tasks"]: - lines.append("| `none` | `pass` | `none` | none | all evidence collected |") + lines.append("| `none` | `pass` | `none` | none | audit gaps closed; ledger validation still required |") for task in plan["tasks"]: lines.extend( [ diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py new file mode 100644 index 00000000..4bd462d4 --- /dev/null +++ b/scripts/world_class_evidence_contract.py @@ -0,0 +1,256 @@ +#!/usr/bin/env python3 +import hashlib +import json +import re +from pathlib import Path +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to share intake validation and artifact integrity checks." + +REQUIRED_TOP_LEVEL = [ + "schema_version", + "evidence_key", + "template_only", + "category", + "source_type", + "submitted_by", + "submitted_at", + "summary", + "artifact_refs", + "provenance", + "privacy", + "anti_overclaim", + "attestation", +] +REQUIRED_PRIVACY_FALSE = [ + "raw_user_content_included", + "raw_provider_prompt_included", + "credentials_included", + "secrets_included", +] +REQUIRED_ANTI_OVERCLAIM_FALSE = [ + "planned_work_counts_as_evidence", + "metadata_fallback_counts_as_native_enforcement", + "pending_review_counts_as_human_decision", + "local_command_runner_counts_as_provider_model", +] +REQUIRED_ATTESTATION_TRUE = [ + "real_external_or_human_evidence", + "reviewer_or_operator_identity_present", + "artifact_refs_reviewed", + "privacy_contract_satisfied", +] +EXPECTED_SOURCE_TYPES = { + "provider-holdout": "provider-output-eval", + "human-adjudication": "blind-ab-review", + "native-permission-enforcement": "runtime-permission-guard", + "native-client-telemetry": "native-client-telemetry", +} +SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") +DISALLOWED_REAL_ARTIFACTS = {"reports/telemetry_events.jsonl"} + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_json_with_status(path: Path) -> tuple[dict[str, Any], str]: + if not path.exists(): + return {}, "missing" + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {}, "invalid-json" + if not isinstance(payload, dict): + return {}, "invalid-json" + return payload, "present" + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def resolve_artifact_path(raw_path: str, root: Path) -> tuple[Path | None, str | None]: + text = raw_path.strip() + if any(token in text for token in ("<", ">", "*", "?")): + return None, "artifact_refs path must be concrete, not a placeholder or glob" + candidate = Path(text) + if candidate.is_absolute(): + return None, "artifact_refs path must be relative to the skill directory" + if ".." in candidate.parts: + return None, "artifact_refs path must not escape the skill directory" + resolved = (root / candidate).resolve() + try: + resolved.relative_to(root.resolve()) + except ValueError: + return None, "artifact_refs path must stay inside the skill directory" + return resolved, None + + +def validate_artifact_refs( + payload: dict[str, Any], + errors: list[str], + *, + root: Path, + template_expected: bool, +) -> dict[str, int]: + refs = payload.get("artifact_refs") + add_error(errors, isinstance(refs, list) and len(refs) > 0, "artifact_refs must contain at least one reference") + stats = { + "artifact_ref_count": len(refs) if isinstance(refs, list) else 0, + "artifact_existing_count": 0, + "artifact_sha256_verified_count": 0, + } + if not isinstance(refs, list): + return stats + for index, ref in enumerate(refs): + if not isinstance(ref, dict): + errors.append(f"artifact_refs[{index}] must be an object") + continue + path_text = str(ref.get("path", "")).strip() + add_error(errors, bool(path_text), f"artifact_refs[{index}].path is required") + add_error(errors, bool(str(ref.get("kind", "")).strip()), f"artifact_refs[{index}].kind is required") + add_error(errors, ref.get("contains_raw_content") is False, f"artifact_refs[{index}] must not contain raw content") + if template_expected or not path_text: + continue + resolved, path_error = resolve_artifact_path(path_text, root) + if path_error: + errors.append(f"artifact_refs[{index}].path {path_error}") + continue + rel = rel_path(resolved, root) + if rel in DISALLOWED_REAL_ARTIFACTS: + errors.append(f"artifact_refs[{index}].path must not reference raw local telemetry logs") + if not resolved.exists() or not resolved.is_file(): + errors.append(f"artifact_refs[{index}].path does not exist as a local file") + continue + stats["artifact_existing_count"] += 1 + declared = str(ref.get("sha256", "")).strip() + if not declared: + errors.append(f"artifact_refs[{index}].sha256 is required for a real submission") + continue + if not SHA256_RE.match(declared): + errors.append(f"artifact_refs[{index}].sha256 must be a 64-character hex digest") + continue + actual = sha256_file(resolved) + if actual.lower() != declared.lower(): + errors.append(f"artifact_refs[{index}].sha256 does not match local artifact") + continue + stats["artifact_sha256_verified_count"] += 1 + return stats + + +def validate_evidence_specific(payload: dict[str, Any], errors: list[str]) -> None: + key = str(payload.get("evidence_key", "")) + provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} + if key == "provider-holdout": + add_error(errors, bool(str(provenance.get("provider", "")).strip()), "provider-holdout provenance.provider is required") + add_error(errors, bool(str(provenance.get("model", "")).strip()), "provider-holdout provenance.model is required") + add_error( + errors, + provenance.get("credential_material_committed") is False, + "provider-holdout must attest credential_material_committed is false", + ) + elif key == "human-adjudication": + add_error(errors, bool(str(provenance.get("reviewer", "")).strip()), "human-adjudication provenance.reviewer is required") + add_error( + errors, + provenance.get("answer_key_opened_after_decisions") is True, + "human-adjudication must attest answer_key_opened_after_decisions is true", + ) + elif key == "native-permission-enforcement": + add_error(errors, bool(str(provenance.get("target", "")).strip()), "native-permission-enforcement provenance.target is required") + add_error( + errors, + bool(str(provenance.get("guard_location", "")).strip()), + "native-permission-enforcement provenance.guard_location is required", + ) + add_error( + errors, + str(provenance.get("guard_scope", "")).strip() + in {"target-client-native", "external-installer-runtime-guard"}, + "native-permission-enforcement provenance.guard_scope must be target-client-native or external-installer-runtime-guard", + ) + add_error( + errors, + provenance.get("guard_blocks_undeclared_capability") is True, + "native-permission-enforcement must attest guard_blocks_undeclared_capability is true", + ) + add_error( + errors, + provenance.get("metadata_fallback_retained_for_other_targets") is True, + "native-permission-enforcement must retain metadata fallback for non-native targets", + ) + elif key == "native-client-telemetry": + add_error(errors, bool(str(provenance.get("client", "")).strip()), "native-client-telemetry provenance.client is required") + add_error(errors, provenance.get("event_source") == "external", "native-client-telemetry event_source must be external") + add_error(errors, provenance.get("metadata_only") is True, "native-client-telemetry must be metadata_only") + + +def validate_payload( + payload: dict[str, Any], + entry: dict[str, Any], + *, + path: Path, + root: Path, + template_expected: bool, +) -> dict[str, Any]: + errors: list[str] = [] + for key in REQUIRED_TOP_LEVEL: + add_error(errors, key in payload, f"missing {key}") + evidence_key = str(entry.get("key", "")) + add_error(errors, payload.get("schema_version") == "1.0", "schema_version must be 1.0") + add_error(errors, payload.get("evidence_key") == evidence_key, f"evidence_key must be {evidence_key}") + add_error(errors, payload.get("template_only") is template_expected, f"template_only must be {str(template_expected).lower()}") + add_error(errors, payload.get("category") == entry.get("category"), f"category must be {entry.get('category')}") + add_error( + errors, + payload.get("source_type") == EXPECTED_SOURCE_TYPES.get(evidence_key), + f"source_type must be {EXPECTED_SOURCE_TYPES.get(evidence_key)}", + ) + add_error(errors, bool(str(payload.get("submitted_by", "")).strip()), "submitted_by is required") + add_error(errors, bool(str(payload.get("submitted_at", "")).strip()), "submitted_at is required") + add_error(errors, bool(str(payload.get("summary", "")).strip()), "summary is required") + artifact_integrity = validate_artifact_refs(payload, errors, root=root, template_expected=template_expected) + privacy = payload.get("privacy", {}) if isinstance(payload.get("privacy", {}), dict) else {} + for key in REQUIRED_PRIVACY_FALSE: + add_error(errors, privacy.get(key) is False, f"privacy.{key} must be false") + anti_overclaim = payload.get("anti_overclaim", {}) if isinstance(payload.get("anti_overclaim", {}), dict) else {} + for key in REQUIRED_ANTI_OVERCLAIM_FALSE: + add_error(errors, anti_overclaim.get(key) is False, f"anti_overclaim.{key} must be false") + validate_evidence_specific(payload, errors) + attestation = payload.get("attestation", {}) if isinstance(payload.get("attestation", {}), dict) else {} + if not template_expected: + for key in REQUIRED_ATTESTATION_TRUE: + add_error(errors, attestation.get(key) is True, f"attestation.{key} must be true for a real submission") + return { + "path": rel_path(path, root), + "evidence_key": evidence_key, + "status": "pass" if not errors else "fail", + "template_only": template_expected, + "artifact_integrity": artifact_integrity, + "errors": errors, + } diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 701cc3c6..b1f21ea5 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -107,6 +107,7 @@ "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", + "scripts/world_class_evidence_contract.py", "scripts/yao.py", "scripts/yao_cli_config.py", "scripts/yao_cli_create_commands.py", diff --git a/tests/verify_trust_check.py b/tests/verify_trust_check.py index ae968679..22d99412 100644 --- a/tests/verify_trust_check.py +++ b/tests/verify_trust_check.py @@ -95,6 +95,7 @@ def main() -> None: "scripts/skill_report_layout.py", "scripts/skill_report_metrics.py", "scripts/skill_report_model.py", + "scripts/world_class_evidence_contract.py", "scripts/yao_cli_config.py", "scripts/yao_cli_parser.py", "scripts/yao_cli_telemetry.py", @@ -109,6 +110,7 @@ def main() -> None: assert "skill_report_layout.py" not in warning_text, payload["warnings"] assert "skill_report_metrics.py" not in warning_text, payload["warnings"] assert "skill_report_model.py" not in warning_text, payload["warnings"] + assert "world_class_evidence_contract.py" not in warning_text, payload["warnings"] assert "yao_cli_config.py" not in warning_text, payload["warnings"] assert "yao_cli_parser.py" not in warning_text, payload["warnings"] assert "yao_cli_telemetry.py" not in warning_text, payload["warnings"] diff --git a/tests/verify_world_class_claim_guard.py b/tests/verify_world_class_claim_guard.py index 6b5a5e33..4562a990 100644 --- a/tests/verify_world_class_claim_guard.py +++ b/tests/verify_world_class_claim_guard.py @@ -52,13 +52,23 @@ def main() -> None: safe_surface = TMP / "safe.md" safe_surface.write_text("ready to claim world-class: `false`\nworld-class evidence is pending.\n", encoding="utf-8") - safe_proc = run_guard("--claim-surface", str(safe_surface), check=True) + safe_proc = run_guard( + "--claim-surface", str(safe_surface), + "--output-json", str(TMP / "safe_guard.json"), + "--output-md", str(TMP / "safe_guard.md"), + check=True, + ) safe_payload = json.loads(safe_proc.stdout) assert safe_payload["summary"]["violation_count"] == 0, safe_payload unsafe_surface = TMP / "unsafe.md" unsafe_surface.write_text("ready to claim world-class: `true`\n世界级已完成\n", encoding="utf-8") - unsafe_proc = run_guard("--claim-surface", str(unsafe_surface), check=False) + unsafe_proc = run_guard( + "--claim-surface", str(unsafe_surface), + "--output-json", str(TMP / "unsafe_guard.json"), + "--output-md", str(TMP / "unsafe_guard.md"), + check=False, + ) assert unsafe_proc.returncode == 2, unsafe_proc.stdout unsafe_payload = json.loads(unsafe_proc.stdout) assert unsafe_payload["ok"] is False, unsafe_payload @@ -72,6 +82,10 @@ def main() -> None: relative_proc = run_guard( "--claim-surface", relative_unsafe_surface.relative_to(ROOT).as_posix(), + "--output-json", + str(TMP / "relative_unsafe_guard.json"), + "--output-md", + str(TMP / "relative_unsafe_guard.md"), check=False, cwd=TMP, ) diff --git a/tests/verify_world_class_evidence_ledger.py b/tests/verify_world_class_evidence_ledger.py index 836929c9..db2d33c0 100644 --- a/tests/verify_world_class_evidence_ledger.py +++ b/tests/verify_world_class_evidence_ledger.py @@ -1,4 +1,5 @@ #!/usr/bin/env python3 +import hashlib import json import shutil import subprocess @@ -11,7 +12,15 @@ SCRIPT = ROOT / "scripts" / "render_world_class_evidence_ledger.py" TMP = ROOT / "tests" / "tmp_world_class_evidence_ledger" -def provider_submission() -> dict: +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def provider_submission(artifact_root: Path = ROOT) -> dict: return { "schema_version": "1.0", "evidence_key": "provider-holdout", @@ -26,8 +35,26 @@ def provider_submission() -> dict: "path": "reports/output_execution_runs.json", "kind": "aggregate-report", "contains_raw_content": False, + "sha256": sha256_file(artifact_root / "reports" / "output_execution_runs.json"), } ], + "provenance": { + "provider": "openai", + "model": "gpt-4.1-mini", + "credential_material_committed": False, + }, + "privacy": { + "raw_user_content_included": False, + "raw_provider_prompt_included": False, + "credentials_included": False, + "secrets_included": False, + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, "attestation": { "real_external_or_human_evidence": True, "reviewer_or_operator_identity_present": True, @@ -136,8 +163,90 @@ def main() -> None: submitted_provider = {entry["key"]: entry for entry in submitted_payload["entries"]}["provider-holdout"] assert submitted_provider["status"] == "pending", submitted_provider assert submitted_provider["submission_state"]["status"] == "submitted", submitted_provider + assert submitted_provider["submission_state"]["artifact_sha256_verified_count"] == 1, submitted_provider assert submitted_provider["submission_state"]["attested_real_evidence"] is True, submitted_provider assert submitted_provider["submission_state"]["ledger_counts_as_completion"] is False, submitted_provider + + accepted_source_skill = TMP / "accepted_source_skill" + (accepted_source_skill / "reports").mkdir(parents=True) + (accepted_source_skill / "reports" / "output_execution_runs.json").write_text( + json.dumps( + { + "summary": { + "model_executed_count": 1, + "timing_observed_count": 1, + "token_observed_count": 1, + } + }, + ensure_ascii=False, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + accepted_source_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "accepted_source_without_submission.json"), + "--output-md", + str(TMP / "accepted_source_without_submission.md"), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + accepted_source_payload = json.loads(accepted_source_proc.stdout) + accepted_source_summary = accepted_source_payload["summary"] + assert accepted_source_summary["source_accepted_count"] == 1, accepted_source_summary + assert accepted_source_summary["accepted_count"] == 0, accepted_source_summary + assert accepted_source_summary["source_accepted_without_valid_submission_count"] == 1, accepted_source_summary + accepted_source_provider = { + entry["key"]: entry for entry in accepted_source_payload["entries"] + }["provider-holdout"] + assert accepted_source_provider["source_accepted"] is True, accepted_source_provider + assert accepted_source_provider["status"] == "pending", accepted_source_provider + assert accepted_source_provider["submission_state"]["status"] == "missing", accepted_source_provider + + accepted_submissions = TMP / "accepted_submissions" + accepted_submissions.mkdir() + (accepted_submissions / "provider-holdout.json").write_text( + json.dumps(provider_submission(accepted_source_skill), ensure_ascii=False, indent=2) + "\n", + encoding="utf-8", + ) + accepted_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(accepted_source_skill), + "--output-json", + str(TMP / "accepted_provider_ledger.json"), + "--output-md", + str(TMP / "accepted_provider_ledger.md"), + "--submissions-dir", + str(accepted_submissions), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + accepted_payload = json.loads(accepted_proc.stdout) + accepted_summary = accepted_payload["summary"] + assert accepted_summary["source_accepted_count"] == 1, accepted_summary + assert accepted_summary["accepted_count"] == 1, accepted_summary + assert accepted_summary["pending_count"] == 3, accepted_summary + assert accepted_summary["source_accepted_without_valid_submission_count"] == 0, accepted_summary + accepted_provider = {entry["key"]: entry for entry in accepted_payload["entries"]}["provider-holdout"] + assert accepted_provider["status"] == "accepted", accepted_provider + assert accepted_provider["submission_state"]["status"] == "submitted", accepted_provider print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_world_class_evidence_plan.py b/tests/verify_world_class_evidence_plan.py index 8c046a78..548eaf1a 100644 --- a/tests/verify_world_class_evidence_plan.py +++ b/tests/verify_world_class_evidence_plan.py @@ -38,18 +38,22 @@ def main() -> None: assert payload["ok"] is True, payload assert payload["summary"]["decision"] == "collect-external-evidence", payload assert payload["summary"]["ready_to_claim_world_class"] is False, payload + assert payload["summary"]["ledger_completion_required"] is True, payload + assert payload["summary"]["evidence_requirement_count"] == 4, payload assert payload["summary"]["task_count"] == 4, payload assert payload["summary"]["human_task_count"] == 1, payload assert payload["summary"]["external_task_count"] == 3, payload assert payload["artifacts"]["ledger"] == "reports/world_class_evidence_ledger.md", payload assert payload["artifacts"]["intake"] == "reports/world_class_evidence_intake.md", payload tasks = {item["key"]: item for item in payload["tasks"]} + requirements = {item["key"]: item for item in payload["evidence_requirements"]} assert set(tasks) == { "provider-holdout", "human-adjudication", "native-permission-enforcement", "native-client-telemetry", }, tasks + assert set(requirements) == set(tasks), requirements assert any("--provider-runner openai" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] assert any("world-class-intake" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] assert any("evidence/world_class/templates/provider-holdout.intake.json" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] @@ -68,6 +72,7 @@ def main() -> None: assert "World-Class Evidence Plan" in markdown, markdown assert "`provider-holdout`" in markdown, markdown assert "ready to claim world-class: `false`" in markdown, markdown + assert "ledger completion required: `true`" in markdown, markdown print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder