From 7c44c9af88f0e89fa5be795db91db170f826bf3e Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 08:19:07 +0800 Subject: [PATCH] Split world-class submission kit verifier --- Makefile | 7 +- reports/adoption_drift_report.json | 2 +- reports/architecture_maintainability.json | 27 +- reports/architecture_maintainability.md | 9 +- reports/benchmark_reproducibility.json | 42 ++- reports/benchmark_reproducibility.md | 21 +- reports/context_budget.json | 14 +- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 8 +- reports/evidence_consistency.json | 34 +-- reports/install_simulation.json | 2 +- reports/output_execution_runs.json | 20 +- reports/output_execution_runs.md | 20 +- reports/python_compatibility.json | 8 +- reports/python_compatibility.md | 2 +- reports/review-studio.html | 16 +- reports/review-studio.json | 104 ++++---- reports/review-viewer.json | 12 +- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/skill-interpretation.json | 12 +- reports/skill-os-2-review.md | 6 +- reports/skill-overview.json | 12 +- reports/skill_os2_audit.json | 2 +- reports/skill_os2_audit.md | 2 +- reports/skill_os2_coverage.json | 2 +- reports/skill_os2_coverage.md | 2 +- reports/skillops/daily/2026-06-16.json | 6 +- reports/skillops/daily/2026-06-16.md | 2 +- reports/skillops/weekly/2026-W25.json | 6 +- reports/skillops/weekly/2026-W25.md | 2 +- reports/world_class_claim_guard.json | 2 +- reports/world_class_claim_guard.md | 2 +- reports/world_class_evidence_intake.json | 2 +- reports/world_class_evidence_intake.md | 2 +- reports/world_class_evidence_ledger.json | 2 +- reports/world_class_evidence_ledger.md | 2 +- reports/world_class_evidence_plan.json | 2 +- reports/world_class_evidence_plan.md | 2 +- reports/world_class_evidence_preflight.json | 2 +- reports/world_class_evidence_preflight.md | 2 +- reports/world_class_operator_runbook.json | 2 +- reports/world_class_operator_runbook.md | 2 +- reports/world_class_submission_review.json | 2 +- reports/world_class_submission_review.md | 2 +- scripts/ci_test.py | 1 + tests/verify_world_class_evidence_intake.py | 226 +--------------- tests/verify_world_class_submission_kit.py | 277 ++++++++++++++++++++ 48 files changed, 517 insertions(+), 423 deletions(-) create mode 100644 tests/verify_world_class_submission_kit.py diff --git a/Makefile b/Makefile index 53b2c105..6461680a 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -116,6 +116,9 @@ world-class-ledger-check: world-class-intake-check: $(PYTHON) tests/verify_world_class_evidence_intake.py +world-class-submission-kit-check: + $(PYTHON) tests/verify_world_class_submission_kit.py + world-class-preflight-check: $(PYTHON) tests/verify_world_class_preflight.py @@ -260,7 +263,7 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 14f57abc..1916fb03 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16T00:06:24Z", + "generated_at": "2026-06-16T00:18:35Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index aa6895cc..bad88abc 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,9 +4,9 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "python_file_count": 191, + "python_file_count": 192, "script_file_count": 122, - "test_file_count": 69, + "test_file_count": 70, "internal_module_count": 39, "cli_script_count": 85, "command_handler_count": 68, @@ -16,7 +16,7 @@ "watch_line_threshold": 720, "block_line_threshold": 1500, "largest_file_lines": 899, - "watchlist_count": 10, + "watchlist_count": 9, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" @@ -29,13 +29,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 874, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/yao_cli_parser.py", "lines": 870, @@ -105,6 +98,13 @@ "kind": "cli-script", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." + }, + { + "path": "scripts/world_class_evidence_contract.py", + "lines": 686, + "kind": "internal-module", + "severity": "pass", + "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." } ], "watchlist": [ @@ -115,13 +115,6 @@ "severity": "pass", "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 874, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/yao_cli_parser.py", "lines": 870, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 3d9b3475..99285d11 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,9 +5,9 @@ Generated at: `2026-06-13` ## Summary - decision: `pass` -- python files: `191` +- python files: `192` - scripts: `122` -- tests: `69` +- tests: `70` - internal modules: `39` - CLI scripts: `85` - Yao CLI command handlers: `68` @@ -15,7 +15,7 @@ Generated at: `2026-06-13` - command modules: `6` - largest file lines: `899` - watch threshold lines: `720` -- watchlist: `10` +- watchlist: `9` - hotspots: `0` - blockers: `0` @@ -30,7 +30,6 @@ No file-size hotspots found. | File | Lines | Kind | Recommended next split | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | -| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | | `tests/verify_review_studio.py` | `836` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. | | `scripts/render_world_class_preflight.py` | `814` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. | @@ -45,7 +44,6 @@ No file-size hotspots found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | | `tests/verify_yao_cli.py` | `899` | `test` | `pass` | -| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | `pass` | | `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` | | `tests/verify_review_studio.py` | `836` | `test` | `pass` | | `scripts/render_world_class_preflight.py` | `814` | `cli-script` | `pass` | @@ -56,6 +54,7 @@ No file-size hotspots found. | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | | `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` | +| `scripts/world_class_evidence_contract.py` | `686` | `internal-module` | `pass` | ## Release Rule diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index bded60d3..a6f125d9 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,21 +3,34 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 22, + "sample": [ + " M Makefile", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md", + " M reports/review-studio.html", + " M reports/review-studio.json", + " M reports/review-viewer.json" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a", + "evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5", "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, @@ -37,14 +50,15 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -53,10 +67,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "ready": false, + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a" + "sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -138,7 +152,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "072e7ff6ef9c938f9b876bd950a69ede0c3d87939fc79f10c0b69b6df250c45d" + "sha256": "dd3e393370f82598b30f0d3de02191a02ff4b596a76463f4995a290517ebb15d" }, { "label": "blind_review", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index b5f96e07..6b1a23b3 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,14 +1,14 @@ # Benchmark Reproducibility Generated at: `2026-06-16` -Commit: `0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a` +Commit: `e74ebe23595710a237e83de176d4fd2e46067824` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `25` - missing artifacts: `0` @@ -22,8 +22,8 @@ Evidence bundle SHA256: `51261159180a2763fe1239729cb6e3da86bf76811560baa342f7420 - world-class ready: `false` - world-class source checks: `7` pass / `13` total; `6` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `22` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | @@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a` +- sha256: `9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5` ## Methodology Sections @@ -72,7 +73,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `072e7ff6ef9c` | +| output_execution | `reports/output_execution_runs.json` | present | `dd3e393370f8` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `bb8c72a9291e` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | diff --git a/reports/context_budget.json b/reports/context_budget.json index f68f25cb..771f0b65 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1046015, + "other_text_tokens": 1046308, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1046812, - "deferred_resource_tokens": 484786, + "estimated_total_text_tokens": 1047105, + "deferred_resource_tokens": 484796, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122 } ], @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 630, + "relevant_file_count": 631, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index 5817f898..c69384aa 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 484786 | `governed` | scripts:424808 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 484796 | `governed` | scripts:424818 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index e4dd2c98..6aa5fec8 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-06-16", + "generated_at": "2026-03-31", "targets": [ { "label": "root", @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 484786, + "deferred_resource_tokens": 484796, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 077eae49..cdbe80dd 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -189,12 +189,12 @@ "status": "pass", "expected": { "status": "pass", - "detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "actual": { "status": "pass", - "detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json" }, "paths": [ @@ -207,8 +207,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": true, - "actual": true, + "expected": false, + "actual": false, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -248,8 +248,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", - "actual": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "expected": "e74ebe23595710a237e83de176d4fd2e46067824", + "actual": "e74ebe23595710a237e83de176d4fd2e46067824", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -261,7 +261,7 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", @@ -271,10 +271,10 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", @@ -284,7 +284,7 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -394,8 +394,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", - "actual": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "expected": "e74ebe23595710a237e83de176d4fd2e46067824", + "actual": "e74ebe23595710a237e83de176d4fd2e46067824", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -407,7 +407,7 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", @@ -417,10 +417,10 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 25, "missing_artifact_count": 0, "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", @@ -430,7 +430,7 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -2411,7 +2411,7 @@ "`25` required artifacts", "`23` reproduction commands", "initial load `990/1000`", - "target count is `80`" + "target count is `81`" ], "actual": "all present", "paths": [ diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 35db4587..c2f80b14 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 647, + "archive_entry_count": 648, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index ec5cf0ca..75d0e39e 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 46.38, + "duration_ms": 42.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 45.36, + "duration_ms": 55.84, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 48.62, + "duration_ms": 43.71, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 43.77, + "duration_ms": 43.07, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 43.21, + "duration_ms": 42.02, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.08, + "duration_ms": 43.49, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 46.12, + "duration_ms": 40.43, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 43.65, + "duration_ms": 46.29, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.54, + "duration_ms": 42.67, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.25, + "duration_ms": 41.76, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 69660ccc..99009c1c 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 46.38 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 45.36 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 48.62 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 43.77 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 43.21 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 40.08 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 46.12 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 43.65 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 40.54 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 40.25 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 42.36 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 55.84 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 43.71 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 43.07 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 42.02 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 43.49 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 40.43 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 46.29 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 42.67 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 41.76 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index cc2eb6eb..eb82d30e 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 194, + "file_count": 195, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -1168,6 +1168,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "tests/verify_world_class_submission_kit.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "tests/verify_world_class_submission_review.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index fa4cccc5..6c0b4f23 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-13` - decision: `pass` - target python: `3.11` -- files scanned: `194` +- files scanned: `195` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/review-studio.html b/reports/review-studio.html index a839c761..2fc74a8f 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -740,12 +740,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

122 scripts scanned; secrets found

Py Compat0

194 files scanned for Python 3.11

Arch Debt0

899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

647 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

122 scripts scanned; secrets found

Py Compat0

194 files scanned for Python 3.11

Arch Debt0

899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock false

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

180 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

647 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
@@ -807,7 +807,7 @@
-

上下文

initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3

+

上下文

initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

@@ -837,14 +837,14 @@
-

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

+

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
-

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
是
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
-

周度队列

决策
curator-review
Week Id
2026-W25
日报数
1
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
是
Writes Source Files
否
Auto Patch Enabled
否
+

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
否
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
+

周度队列

决策
curator-review
Week Id
2026-W25
日报数
1
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
否
Writes Source Files
否
Auto Patch Enabled
否
@@ -877,8 +877,8 @@
-

公开声明

本地复现
是
发布锁
是
可公开声明
否
声明阻断
4
Provider 证据
否
人审完成
否
世界级就绪
否
-

声明阻断

  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (7/13 pass, 6 blocked)
+

公开声明

本地复现
是
发布锁
否
可公开声明
否
声明阻断
5
Provider 证据
否
人审完成
否
世界级就绪
否
+

声明阻断

  • 阻断release lock is not clean or commit is unavailable
  • 阻断provider-backed model holdout evidence is incomplete
  • 阻断human blind-review adjudication is incomplete
  • 阻断world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)
  • 阻断world-class source checks are not all accepted (7/13 pass, 6 blocked)
diff --git a/reports/review-studio.json b/reports/review-studio.json index 45938069..5d37721d 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -43,7 +43,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -107,7 +107,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", + "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false", "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", "link": "adoption_drift_report.md" }, @@ -1913,11 +1913,11 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a", + "evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5", "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, @@ -1937,11 +1937,11 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -4593,7 +4593,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 46.38, + "duration_ms": 42.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4621,7 +4621,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 45.36, + "duration_ms": 55.84, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4644,7 +4644,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 48.62, + "duration_ms": 43.71, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4672,7 +4672,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 43.77, + "duration_ms": 43.07, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4695,7 +4695,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 43.21, + "duration_ms": 42.02, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4723,7 +4723,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.08, + "duration_ms": 43.49, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4746,7 +4746,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 46.12, + "duration_ms": 40.43, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4773,7 +4773,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 43.65, + "duration_ms": 46.29, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4796,7 +4796,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.54, + "duration_ms": 42.67, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4825,7 +4825,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 40.25, + "duration_ms": 41.76, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5551,21 +5551,34 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 22, + "sample": [ + " M Makefile", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/evidence_consistency.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md", + " M reports/review-studio.html", + " M reports/review-studio.json", + " M reports/review-viewer.json" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a", + "evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5", "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, @@ -5585,14 +5598,15 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -5601,10 +5615,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "ready": false, + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -5612,7 +5626,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a" + "sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5686,7 +5700,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "072e7ff6ef9c938f9b876bd950a69ede0c3d87939fc79f10c0b69b6df250c45d" + "sha256": "dd3e393370f82598b30f0d3de02191a02ff4b596a76463f4995a290517ebb15d" }, { "label": "blind_review", @@ -16350,15 +16364,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1046015, + "other_text_tokens": 1046381, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1046812, - "deferred_resource_tokens": 484786, + "estimated_total_text_tokens": 1047178, + "deferred_resource_tokens": 484796, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122 }, { @@ -16380,7 +16394,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122 } ], @@ -16403,14 +16417,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 424808, + "estimated_tokens": 424818, "file_count": 122, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 630, + "relevant_file_count": 631, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 @@ -18394,7 +18408,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -18411,7 +18425,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -18860,7 +18874,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true }, "actions": [ @@ -18909,7 +18923,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -18929,7 +18943,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -19181,7 +19195,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 4b9b687f..717cc182 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -996,11 +996,11 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a", + "evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5", "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, @@ -1020,11 +1020,11 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index ce7f483e..946398b7 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 213, - "package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f" + "package_sha256": "5599cd15154d9fa948e9907f5ada8e22486166be102739eed26e0a1fdad3a326" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index be4ab92f..61564b79 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `213` -- Package SHA256: `a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f` +- Package SHA256: `5599cd15154d9fa948e9907f5ada8e22486166be102739eed26e0a1fdad3a326` ## Failures diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index c1f4d8e7..1aec5cdb 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -1000,11 +1000,11 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a", + "evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5", "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, @@ -1024,11 +1024,11 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index bfbe2b70..9c7e5caf 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -66,8 +66,8 @@ This is still not the final world-class state. Target-native behavior contracts | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | | World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | -| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename and nested raw-field rejection | v0 landed | -| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_evidence_intake.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed | +| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, and nested raw-field rejection | v0 landed | +| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | | Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed | @@ -146,7 +146,7 @@ Next move: add real client or installer permission enforcement integration. | Benchmark Reproducibility | local reproducibility ready with `25` required artifacts, `0` missing artifacts, `23` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations | | IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable | | Context Budget | initial load `990/1000`, under the production budget | -| CI | `make ci-test` target count is `80` after the Weekly SkillOps Curator gate update | +| CI | `make ci-test` target count is `81` after the World-Class Submission Kit verifier split | ## Next Highest-Leverage Moves diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 323a6aaf..9d56b57d 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -995,11 +995,11 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a", + "evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5", "source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f", "archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81", "output_case_count": 5, @@ -1019,11 +1019,11 @@ "world_class_source_pass_count": 7, "world_class_source_blocked_count": 6, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 22 }, - "commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8", + "commit": "e74ebe23595710a237e83de176d4fd2e46067824", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 8559a597..81907e02 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "item_count": 15, diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 1bf27b9b..cdbb81db 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -1,6 +1,6 @@ # Skill OS 2.0 Audit -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 32cfeec2..08a7e553 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "item_count": 21, diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 35b3256f..bbb21e83 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -1,6 +1,6 @@ # Skill OS 2.0 Blueprint Coverage -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/skillops/daily/2026-06-16.json b/reports/skillops/daily/2026-06-16.json index a35cbc88..386b0613 100644 --- a/reports/skillops/daily/2026-06-16.json +++ b/reports/skillops/daily/2026-06-16.json @@ -14,7 +14,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -31,7 +31,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -480,7 +480,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true }, "actions": [ diff --git a/reports/skillops/daily/2026-06-16.md b/reports/skillops/daily/2026-06-16.md index 105c41c3..d0fbea0f 100644 --- a/reports/skillops/daily/2026-06-16.md +++ b/reports/skillops/daily/2026-06-16.md @@ -14,7 +14,7 @@ Generated at: `2026-06-16` - local blueprint ready: `true` - public world-class ready: `false` - world-class pending: `4` -- release lock ready: `true` +- release lock ready: `false` - evidence consistency ok: `true` This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence. diff --git a/reports/skillops/weekly/2026-W25.json b/reports/skillops/weekly/2026-W25.json index 998d23ea..65c79eb3 100644 --- a/reports/skillops/weekly/2026-W25.json +++ b/reports/skillops/weekly/2026-W25.json @@ -15,7 +15,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -35,7 +35,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -287,7 +287,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 diff --git a/reports/skillops/weekly/2026-W25.md b/reports/skillops/weekly/2026-W25.md index 4d75fc31..1fd6a716 100644 --- a/reports/skillops/weekly/2026-W25.md +++ b/reports/skillops/weekly/2026-W25.md @@ -13,7 +13,7 @@ Week: `2026-W25` - top_score: `88` - skill_count: `12` - actionable_portfolio_issue_count: `0` -- release_lock_ready: `True` +- release_lock_ready: `False` - evidence_consistency_ok: `True` - public_world_class_ready: `False` - world_class_pending_count: `4` diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index 18f8d5e4..d67c56c7 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "ledger_ready_to_claim_world_class": false, diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index e33c9532..c389e495 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -1,6 +1,6 @@ # World-Class Claim Guard -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index f97f646f..df978d9e 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "schema_present": true, diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index c5a01fae..297677c7 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -1,6 +1,6 @@ # World-Class Evidence Intake -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index 3d24c166..34a900c1 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "decision": "evidence-pending", "ready_to_claim_world_class": false, diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index 420fbb52..94cd2079 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -1,6 +1,6 @@ # World-Class Evidence Ledger -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index 845c0ad8..50908b39 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "audit_decision": "continue-iteration", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index 8c5c9537..3a1b0962 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -1,6 +1,6 @@ # World-Class Evidence Plan -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index 615577db..4a7eaf7a 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "evidence_item_count": 4, diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index b2349713..e487d321 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -1,6 +1,6 @@ # World-Class Evidence Preflight -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index bc72296d..6108ef7e 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "evidence_item_count": 4, diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index 9db0580c..8ba17d83 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -1,6 +1,6 @@ # World-Class Operator Runbook -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 8eaeeee6..8abee31b 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", "decision": "awaiting-submissions", "ready_to_claim_world_class": false, diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 322fc7ff..21bd5b52 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -1,6 +1,6 @@ # World-Class Submission Review -Generated at: `2026-06-16` +Generated at: `2026-06-13` ## Summary diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 0e779d4a..e214c5d3 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -59,6 +59,7 @@ DEFAULT_TARGETS = [ "world-class-evidence-check", "world-class-ledger-check", "world-class-intake-check", + "world-class-submission-kit-check", "world-class-preflight-check", "world-class-submission-review-check", "world-class-runbook-check", diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 269b82bb..a1c83b5b 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -9,7 +9,6 @@ from pathlib import Path ROOT = Path(__file__).resolve().parent.parent SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py" -KIT_SCRIPT = ROOT / "scripts" / "prepare_world_class_submission_kit.py" TMP = ROOT / "tests" / "tmp_world_class_evidence_intake" sys.path.insert(0, str(ROOT / "scripts")) @@ -25,6 +24,11 @@ def sha256_file(path: Path) -> str: def run_intake(*extra: str) -> dict: + extra_args = list(extra) + if "--output-json" not in extra_args: + extra_args.extend(["--output-json", str(TMP / "last_intake.json")]) + if "--output-md" not in extra_args: + extra_args.extend(["--output-md", str(TMP / "last_intake.md")]) proc = subprocess.run( [ sys.executable, @@ -32,7 +36,7 @@ def run_intake(*extra: str) -> dict: str(ROOT), "--generated-at", "2026-06-14", - *extra, + *extra_args, ], cwd=ROOT, capture_output=True, @@ -42,23 +46,6 @@ def run_intake(*extra: str) -> dict: return json.loads(proc.stdout) -def run_kit(*extra: str) -> subprocess.CompletedProcess[str]: - return subprocess.run( - [ - sys.executable, - str(KIT_SCRIPT), - str(ROOT), - "--generated-at", - "2026-06-14", - *extra, - ], - cwd=ROOT, - capture_output=True, - text=True, - check=True, - ) - - def provider_submission(*, valid: bool, artifact_path: str = "reports/output_execution_runs.json") -> dict: return { "schema_version": "1.0", @@ -516,27 +503,9 @@ def assert_external_contract_artifact_validation() -> None: assert any("summary.adoption_sample_count must be >0" in error for error in telemetry_invalid["errors"]), telemetry_invalid["errors"] -def assert_documented_submission_commands() -> None: - expected_fragments = [ - 'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"', - 'world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR"', - 'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR"', - 'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" --prefill-artifacts', - 'world-class-intake . --submissions-dir "$SUBMISSIONS_DIR"', - 'world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR"', - 'world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR"', - ] - for relative_path in ("README.md", "evidence/world_class/README.md"): - text = (ROOT / relative_path).read_text(encoding="utf-8") - for fragment in expected_fragments: - assert fragment in text, (relative_path, fragment) - assert "/tmp/yao-world-class-submission-kit" not in text, relative_path - - def main() -> None: shutil.rmtree(TMP, ignore_errors=True) TMP.mkdir(parents=True, exist_ok=True) - assert_documented_submission_commands() default_json = TMP / "world_class_evidence_intake.json" default_md = TMP / "world_class_evidence_intake.md" assert_human_contract_artifact_validation() @@ -610,189 +579,6 @@ def main() -> None: assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown - kit_dir = TMP / "submission_kit" - kit_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout") - kit_payload = json.loads(kit_proc.stdout) - assert kit_payload["ok"] is True, kit_payload - assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"] - assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"] - assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"] - assert kit_payload["summary"]["submission_ref_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["submission_ref_ready_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["supporting_artifact_count"] >= 1, kit_payload["summary"] - assert kit_payload["summary"]["supporting_artifact_ready_count"] >= 1, kit_payload["summary"] - assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"] - assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"] - assert kit_payload["summary"]["evidence_matrix_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["evidence_matrix_collect_source_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["evidence_matrix_submission_ref_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["evidence_matrix_submission_ref_ready_count"] == 1, kit_payload["summary"] - assert kit_payload["summary"]["evidence_matrix_supporting_artifact_count"] >= 1, kit_payload["summary"] - assert kit_payload["summary"]["evidence_matrix_supporting_artifact_ready_count"] >= 1, kit_payload["summary"] - assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"] - assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] - assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] - assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"] - assert kit_payload["files"][0]["output_path"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/provider-holdout.json"), kit_payload["files"] - assert len(kit_payload["evidence_matrix"]) == 1, kit_payload["evidence_matrix"] - matrix_row = kit_payload["evidence_matrix"][0] - assert matrix_row["evidence_key"] == "provider-holdout", matrix_row - assert matrix_row["stage"] == "collect-source", matrix_row - assert matrix_row["draft_status"] == "written", matrix_row - assert matrix_row["artifact_ready_count"] >= 1, matrix_row - assert matrix_row["artifact_total_count"] >= matrix_row["artifact_ready_count"], matrix_row - assert matrix_row["submission_ref_ready_count"] == 1, matrix_row - assert matrix_row["submission_ref_total_count"] == 1, matrix_row - assert matrix_row["supporting_artifact_ready_count"] >= 1, matrix_row - assert matrix_row["supporting_artifact_total_count"] >= matrix_row["supporting_artifact_ready_count"], matrix_row - assert matrix_row["source_pass_count"] == 1, matrix_row - assert matrix_row["source_check_count"] == 3, matrix_row - assert matrix_row["source_blocked_count"] == 2, matrix_row - assert matrix_row["counts_as_completion"] is False, matrix_row - assert "real credentials" in matrix_row["next_action"], matrix_row - artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]} - assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows - assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows - assert artifact_rows["reports/output_execution_runs.json"]["artifact_role"] == "submission-ref", artifact_rows - assert artifact_rows["reports/output_execution_runs.json"]["submission_ref_required"] is True, artifact_rows - assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows - assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows - assert artifact_rows["reports/output_execution_runs.md"]["artifact_role"] == "supporting-evidence", artifact_rows - assert artifact_rows["reports/output_execution_runs.md"]["submission_ref_required"] is False, artifact_rows - source_rows = {item["field"]: item for item in kit_payload["source_checklist"]} - assert source_rows["model_executed_count"]["status"] == "blocked", source_rows - assert source_rows["model_executed_count"]["actual"] == 0, source_rows - assert source_rows["timing_observed_count"]["status"] == "pass", source_rows - assert source_rows["token_observed_count"]["status"] == "blocked", source_rows - kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) - assert kit_draft["template_only"] is True, kit_draft - assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft - assert "sha256" not in kit_draft["artifact_refs"][0], kit_draft - assert kit_draft["provenance"]["run_command"] == "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", kit_draft - assert kit_draft["provenance"]["credential_env"] == "OPENAI_API_KEY", kit_draft - assert "" not in json.dumps(kit_draft), kit_draft - kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8")) - assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"] - assert kit_manifest["summary"]["artifact_prefill_enabled"] is False, kit_manifest["summary"] - assert kit_manifest["summary"]["artifact_ref_prefill_count"] == 0, kit_manifest["summary"] - assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"] - assert kit_manifest["source_checklist"] == kit_payload["source_checklist"], kit_manifest["source_checklist"] - assert kit_manifest["artifacts"]["html"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/index.html"), kit_manifest["artifacts"] - kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8") - assert "Drafts are not accepted evidence." in kit_readme, kit_readme - assert "Execution Runbook" in kit_readme, kit_readme - assert "output-exec --provider-runner openai" in kit_readme, kit_readme - assert "" not in kit_readme, kit_readme - assert "validate intake" in kit_readme, kit_readme - assert "Artifact Checklist" in kit_readme, kit_readme - assert "Evidence Matrix" in kit_readme, kit_readme - assert "Submission refs" in kit_readme, kit_readme - assert "Supporting assets" in kit_readme, kit_readme - assert "`submission-ref`" in kit_readme, kit_readme - assert "`supporting-evidence`" in kit_readme, kit_readme - assert "`collect-source`" in kit_readme, kit_readme - assert "Matrix rows are guidance only" in kit_readme, kit_readme - assert "Source Evidence Snapshot" in kit_readme, kit_readme - assert "reports/output_execution_runs.json" in kit_readme, kit_readme - assert "Provider model run" in kit_readme, kit_readme - kit_html = (kit_dir / "index.html").read_text(encoding="utf-8") - assert "World-Class Evidence Submission Kit" in kit_html, kit_html - assert "Drafts are not accepted evidence" in kit_html, kit_html - assert "provider-holdout" in kit_html, kit_html - assert "Artifact Checklist" in kit_html, kit_html - assert "Evidence Matrix" in kit_html, kit_html - assert "matrix-card collect-source" in kit_html, kit_html - assert ( - f"
Submission refs
{matrix_row['submission_ref_ready_count']}/{matrix_row['submission_ref_total_count']} ready
" - in kit_html - ), kit_html - assert ( - f"
Supporting assets
{matrix_row['supporting_artifact_ready_count']}/{matrix_row['supporting_artifact_total_count']} ready
" - in kit_html - ), kit_html - assert "
Source
1/3 pass
" in kit_html, kit_html - assert "Source Evidence Snapshot" in kit_html, kit_html - assert "
Field
model_executed_count
" in kit_html, kit_html - assert "
Current
0
" in kit_html, kit_html - assert "
Expected
>0
" in kit_html, kit_html - assert "World-Class Evidence Submission Kit" in kit_html, kit_html - assert "Execution Runbook" in kit_html, kit_html - assert "output-exec --provider-runner openai" in kit_html, kit_html - assert "<redacted>" not in kit_html and "" not in kit_html, kit_html - assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html - assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html - - prefilled_kit_dir = TMP / "prefilled_submission_kit" - prefilled_proc = run_kit( - "--output-dir", - str(prefilled_kit_dir), - "--evidence-key", - "provider-holdout", - "--prefill-artifacts", - ) - prefilled_payload = json.loads(prefilled_proc.stdout) - assert prefilled_payload["summary"]["artifact_prefill_enabled"] is True, prefilled_payload["summary"] - assert prefilled_payload["summary"]["artifact_ref_prefill_count"] == 1, prefilled_payload["summary"] - assert prefilled_payload["summary"]["artifact_ref_unfilled_count"] == 0, prefilled_payload["summary"] - assert prefilled_payload["summary"]["drafts_count_as_evidence"] is False, prefilled_payload["summary"] - assert prefilled_payload["safety"]["artifact_prefill_counts_as_evidence"] is False, prefilled_payload["safety"] - assert prefilled_payload["files"][0]["prefilled_artifact_ref_count"] == 1, prefilled_payload["files"] - prefilled_draft = json.loads((prefilled_kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) - assert prefilled_draft["template_only"] is True, prefilled_draft - assert prefilled_draft["attestation"]["real_external_or_human_evidence"] is False, prefilled_draft - assert len(prefilled_draft["artifact_refs"][0]["sha256"]) == 64, prefilled_draft - assert prefilled_draft["artifact_refs"][0]["contains_raw_content"] is False, prefilled_draft - prefilled_readme = (prefilled_kit_dir / "README.md").read_text(encoding="utf-8") - assert "Optional artifact prefill only inserts SHA-256 digests" in prefilled_readme, prefilled_readme - assert "does not mark a draft as real evidence" in prefilled_readme, prefilled_readme - prefilled_html = (prefilled_kit_dir / "index.html").read_text(encoding="utf-8") - assert "artifact prefill only inserts local SHA-256 digests" in prefilled_html, prefilled_html - assert "
Prefill
1 artifact refs
" in prefilled_html, prefilled_html - - native_kit_dir = TMP / "native_permission_kit" - native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement") - native_kit_payload = json.loads(native_kit_proc.stdout) - native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]} - assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 1, native_kit_payload["summary"] - if "dist/targets/openai/adapter.json" in native_rows: - assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows - assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows - assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows - else: - glob_rows = [ - item for item in native_kit_payload["artifact_checklist"] - if item["source_pattern"] == "dist/targets/*/adapter.json" - ] - assert len(glob_rows) == 1, glob_rows - assert glob_rows[0]["status"] == "glob-no-match", glob_rows[0] - assert glob_rows[0]["artifact_ref_ready"] is False, glob_rows[0] - assert glob_rows[0]["concrete_reference_required"] is True, glob_rows[0] - native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8") - assert "Glob patterns are expanded into concrete files" in native_readme, native_readme - draft_intake = run_intake("--submissions-dir", str(kit_dir)) - assert draft_intake["ok"] is False, draft_intake - assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"] - assert draft_intake["summary"]["invalid_submission_count"] == 1, draft_intake["summary"] - assert draft_intake["submissions"][0]["evidence_key"] == "provider-holdout", draft_intake["submissions"] - assert all(item["evidence_key"] != "unknown" for item in draft_intake["submissions"]), draft_intake["submissions"] - - spaced_kit_dir = TMP / "submission kit spaced" - spaced_kit_proc = run_kit("--output-dir", str(spaced_kit_dir), "--evidence-key", "provider-holdout") - spaced_kit_payload = json.loads(spaced_kit_proc.stdout) - quoted_spaced_dir = "'tests/tmp_world_class_evidence_intake/submission kit spaced'" - assert quoted_spaced_dir in spaced_kit_payload["commands"]["validate_intake"], spaced_kit_payload["commands"] - assert quoted_spaced_dir in spaced_kit_payload["commands"]["refresh_ledger"], spaced_kit_payload["commands"] - spaced_readme = (spaced_kit_dir / "README.md").read_text(encoding="utf-8") - assert quoted_spaced_dir in spaced_readme, spaced_readme - - existing_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout") - existing_payload = json.loads(existing_proc.stdout) - assert existing_payload["summary"]["existing_count"] == 1, existing_payload["summary"] - assert existing_payload["files"][0]["status"] == "exists", existing_payload["files"] - valid_dir = TMP / "valid_submissions" valid_dir.mkdir() (valid_dir / "provider-holdout.json").write_text( diff --git a/tests/verify_world_class_submission_kit.py b/tests/verify_world_class_submission_kit.py new file mode 100644 index 00000000..60fc5795 --- /dev/null +++ b/tests/verify_world_class_submission_kit.py @@ -0,0 +1,277 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +INTAKE_SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py" +KIT_SCRIPT = ROOT / "scripts" / "prepare_world_class_submission_kit.py" +TMP = ROOT / "tests" / "tmp_world_class_submission_kit" + + +def run_intake(*extra: str) -> dict: + extra_args = list(extra) + if "--output-json" not in extra_args: + extra_args.extend(["--output-json", str(TMP / "last_intake.json")]) + if "--output-md" not in extra_args: + extra_args.extend(["--output-md", str(TMP / "last_intake.md")]) + proc = subprocess.run( + [ + sys.executable, + str(INTAKE_SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra_args, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + return json.loads(proc.stdout) + + +def run_kit(*extra: str) -> subprocess.CompletedProcess[str]: + return subprocess.run( + [ + sys.executable, + str(KIT_SCRIPT), + str(ROOT), + "--generated-at", + "2026-06-14", + *extra, + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + + +def assert_documented_submission_commands() -> None: + expected_fragments = [ + 'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"', + 'world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR"', + 'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR"', + 'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" --prefill-artifacts', + 'world-class-intake . --submissions-dir "$SUBMISSIONS_DIR"', + 'world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR"', + 'world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR"', + ] + for relative_path in ("README.md", "evidence/world_class/README.md"): + text = (ROOT / relative_path).read_text(encoding="utf-8") + for fragment in expected_fragments: + assert fragment in text, (relative_path, fragment) + assert "/tmp/yao-world-class-submission-kit" not in text, relative_path + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + assert_documented_submission_commands() + + kit_dir = TMP / "submission_kit" + kit_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout") + kit_payload = json.loads(kit_proc.stdout) + assert kit_payload["ok"] is True, kit_payload + assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"] + assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["submission_ref_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["submission_ref_ready_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["supporting_artifact_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["supporting_artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"] + assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_collect_source_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_submission_ref_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_submission_ref_ready_count"] == 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_supporting_artifact_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_supporting_artifact_ready_count"] >= 1, kit_payload["summary"] + assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"] + assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"] + assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"] + assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"] + assert kit_payload["files"][0]["output_path"].endswith( + f"tests/{TMP.name}/submission_kit/provider-holdout.json" + ), kit_payload["files"] + + assert len(kit_payload["evidence_matrix"]) == 1, kit_payload["evidence_matrix"] + matrix_row = kit_payload["evidence_matrix"][0] + assert matrix_row["evidence_key"] == "provider-holdout", matrix_row + assert matrix_row["stage"] == "collect-source", matrix_row + assert matrix_row["draft_status"] == "written", matrix_row + assert matrix_row["artifact_ready_count"] >= 1, matrix_row + assert matrix_row["artifact_total_count"] >= matrix_row["artifact_ready_count"], matrix_row + assert matrix_row["submission_ref_ready_count"] == 1, matrix_row + assert matrix_row["submission_ref_total_count"] == 1, matrix_row + assert matrix_row["supporting_artifact_ready_count"] >= 1, matrix_row + assert matrix_row["supporting_artifact_total_count"] >= matrix_row["supporting_artifact_ready_count"], matrix_row + assert matrix_row["source_pass_count"] == 1, matrix_row + assert matrix_row["source_check_count"] == 3, matrix_row + assert matrix_row["source_blocked_count"] == 2, matrix_row + assert matrix_row["counts_as_completion"] is False, matrix_row + assert "real credentials" in matrix_row["next_action"], matrix_row + + artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]} + assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["artifact_role"] == "submission-ref", artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["submission_ref_required"] is True, artifact_rows + assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows + assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows + assert artifact_rows["reports/output_execution_runs.md"]["artifact_role"] == "supporting-evidence", artifact_rows + assert artifact_rows["reports/output_execution_runs.md"]["submission_ref_required"] is False, artifact_rows + + source_rows = {item["field"]: item for item in kit_payload["source_checklist"]} + assert source_rows["model_executed_count"]["status"] == "blocked", source_rows + assert source_rows["model_executed_count"]["actual"] == 0, source_rows + assert source_rows["timing_observed_count"]["status"] == "pass", source_rows + assert source_rows["token_observed_count"]["status"] == "blocked", source_rows + + kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) + assert kit_draft["template_only"] is True, kit_draft + assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft + assert "sha256" not in kit_draft["artifact_refs"][0], kit_draft + assert kit_draft["provenance"]["run_command"] == ( + "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60" + ), kit_draft + assert kit_draft["provenance"]["credential_env"] == "OPENAI_API_KEY", kit_draft + assert "" not in json.dumps(kit_draft), kit_draft + + kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8")) + assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"] + assert kit_manifest["summary"]["artifact_prefill_enabled"] is False, kit_manifest["summary"] + assert kit_manifest["summary"]["artifact_ref_prefill_count"] == 0, kit_manifest["summary"] + assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"] + assert kit_manifest["source_checklist"] == kit_payload["source_checklist"], kit_manifest["source_checklist"] + assert kit_manifest["artifacts"]["html"].endswith( + f"tests/{TMP.name}/submission_kit/index.html" + ), kit_manifest["artifacts"] + + kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8") + assert "Drafts are not accepted evidence." in kit_readme, kit_readme + assert "Execution Runbook" in kit_readme, kit_readme + assert "output-exec --provider-runner openai" in kit_readme, kit_readme + assert "" not in kit_readme, kit_readme + assert "validate intake" in kit_readme, kit_readme + assert "Artifact Checklist" in kit_readme, kit_readme + assert "Evidence Matrix" in kit_readme, kit_readme + assert "Submission refs" in kit_readme, kit_readme + assert "Supporting assets" in kit_readme, kit_readme + assert "`submission-ref`" in kit_readme, kit_readme + assert "`supporting-evidence`" in kit_readme, kit_readme + assert "`collect-source`" in kit_readme, kit_readme + assert "Matrix rows are guidance only" in kit_readme, kit_readme + assert "Source Evidence Snapshot" in kit_readme, kit_readme + assert "reports/output_execution_runs.json" in kit_readme, kit_readme + assert "Provider model run" in kit_readme, kit_readme + + kit_html = (kit_dir / "index.html").read_text(encoding="utf-8") + assert "World-Class Evidence Submission Kit" in kit_html, kit_html + assert "Drafts are not accepted evidence" in kit_html, kit_html + assert "provider-holdout" in kit_html, kit_html + assert "Artifact Checklist" in kit_html, kit_html + assert "Evidence Matrix" in kit_html, kit_html + assert "matrix-card collect-source" in kit_html, kit_html + assert ( + f"
Submission refs
{matrix_row['submission_ref_ready_count']}/{matrix_row['submission_ref_total_count']} ready
" + in kit_html + ), kit_html + assert ( + f"
Supporting assets
{matrix_row['supporting_artifact_ready_count']}/{matrix_row['supporting_artifact_total_count']} ready
" + in kit_html + ), kit_html + assert "
Source
1/3 pass
" in kit_html, kit_html + assert "Source Evidence Snapshot" in kit_html, kit_html + assert "
Field
model_executed_count
" in kit_html, kit_html + assert "
Current
0
" in kit_html, kit_html + assert "
Expected
>0
" in kit_html, kit_html + assert "World-Class Evidence Submission Kit" in kit_html, kit_html + assert "Execution Runbook" in kit_html, kit_html + assert "output-exec --provider-runner openai" in kit_html, kit_html + assert "<redacted>" not in kit_html and "" not in kit_html, kit_html + assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html + assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html + + prefilled_kit_dir = TMP / "prefilled_submission_kit" + prefilled_proc = run_kit( + "--output-dir", + str(prefilled_kit_dir), + "--evidence-key", + "provider-holdout", + "--prefill-artifacts", + ) + prefilled_payload = json.loads(prefilled_proc.stdout) + assert prefilled_payload["summary"]["artifact_prefill_enabled"] is True, prefilled_payload["summary"] + assert prefilled_payload["summary"]["artifact_ref_prefill_count"] == 1, prefilled_payload["summary"] + assert prefilled_payload["summary"]["artifact_ref_unfilled_count"] == 0, prefilled_payload["summary"] + assert prefilled_payload["summary"]["drafts_count_as_evidence"] is False, prefilled_payload["summary"] + assert prefilled_payload["safety"]["artifact_prefill_counts_as_evidence"] is False, prefilled_payload["safety"] + assert prefilled_payload["files"][0]["prefilled_artifact_ref_count"] == 1, prefilled_payload["files"] + prefilled_draft = json.loads((prefilled_kit_dir / "provider-holdout.json").read_text(encoding="utf-8")) + assert prefilled_draft["template_only"] is True, prefilled_draft + assert prefilled_draft["attestation"]["real_external_or_human_evidence"] is False, prefilled_draft + assert len(prefilled_draft["artifact_refs"][0]["sha256"]) == 64, prefilled_draft + assert prefilled_draft["artifact_refs"][0]["contains_raw_content"] is False, prefilled_draft + prefilled_readme = (prefilled_kit_dir / "README.md").read_text(encoding="utf-8") + assert "Optional artifact prefill only inserts SHA-256 digests" in prefilled_readme, prefilled_readme + assert "does not mark a draft as real evidence" in prefilled_readme, prefilled_readme + prefilled_html = (prefilled_kit_dir / "index.html").read_text(encoding="utf-8") + assert "artifact prefill only inserts local SHA-256 digests" in prefilled_html, prefilled_html + assert "
Prefill
1 artifact refs
" in prefilled_html, prefilled_html + + native_kit_dir = TMP / "native_permission_kit" + native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement") + native_kit_payload = json.loads(native_kit_proc.stdout) + native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]} + assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 1, native_kit_payload["summary"] + if "dist/targets/openai/adapter.json" in native_rows: + assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows + assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows + assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows + else: + glob_rows = [ + item for item in native_kit_payload["artifact_checklist"] + if item["source_pattern"] == "dist/targets/*/adapter.json" + ] + assert len(glob_rows) == 1, glob_rows + assert glob_rows[0]["status"] == "glob-no-match", glob_rows[0] + assert glob_rows[0]["artifact_ref_ready"] is False, glob_rows[0] + assert glob_rows[0]["concrete_reference_required"] is True, glob_rows[0] + native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8") + assert "Glob patterns are expanded into concrete files" in native_readme, native_readme + + draft_intake = run_intake("--submissions-dir", str(kit_dir)) + assert draft_intake["ok"] is False, draft_intake + assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"] + assert draft_intake["summary"]["invalid_submission_count"] == 1, draft_intake["summary"] + assert draft_intake["submissions"][0]["evidence_key"] == "provider-holdout", draft_intake["submissions"] + assert all(item["evidence_key"] != "unknown" for item in draft_intake["submissions"]), draft_intake["submissions"] + + spaced_kit_dir = TMP / "submission kit spaced" + spaced_kit_proc = run_kit("--output-dir", str(spaced_kit_dir), "--evidence-key", "provider-holdout") + spaced_kit_payload = json.loads(spaced_kit_proc.stdout) + quoted_spaced_dir = f"'tests/{TMP.name}/submission kit spaced'" + assert quoted_spaced_dir in spaced_kit_payload["commands"]["validate_intake"], spaced_kit_payload["commands"] + assert quoted_spaced_dir in spaced_kit_payload["commands"]["refresh_ledger"], spaced_kit_payload["commands"] + spaced_readme = (spaced_kit_dir / "README.md").read_text(encoding="utf-8") + assert quoted_spaced_dir in spaced_readme, spaced_readme + + existing_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout") + existing_payload = json.loads(existing_proc.stdout) + assert existing_payload["summary"]["existing_count"] == 1, existing_payload["summary"] + assert existing_payload["files"][0]["status"] == "exists", existing_payload["files"] + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main()