frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Split world-class submission kit verifier

Browse Source
This commit is contained in:
yaojingang
2026-06-16 08:19:07 +08:00
parent e74ebe2359
commit 7c44c9af88
48 changed files with 517 additions and 423 deletions
Download Patch File Download Diff File Expand all files Collapse all files
Makefile
+5 -2
View File
@@ -2,7 +2,7 @@ PYTHON ?= python3
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-kit-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
eval:
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
@@ -116,6 +116,9 @@ world-class-ledger-check:
world-class-intake-check:
$(PYTHON) tests/verify_world_class_evidence_intake.py
world-class-submission-kit-check:
$(PYTHON) tests/verify_world_class_submission_kit.py
world-class-preflight-check:
$(PYTHON) tests/verify_world_class_preflight.py
@@ -260,7 +263,7 @@ sync-local-install: package-check
sync-active-install: package-check
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check python-compat-check architecture-maintainability-check yao-cli-check skill-overview-check skill-interpretation-check skill-report-metrics-check skill-report-charts-check html-rendering-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check skill-os2-coverage-check world-class-evidence-check world-class-ledger-check world-class-intake-check world-class-submission-kit-check world-class-preflight-check world-class-submission-review-check world-class-runbook-check world-class-claim-guard-check benchmark-reproducibility-check evidence-consistency-check feedback-check adaptation-safety-check skillops-opportunity-check daily-skillops-check weekly-curator-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
ci-test:
$(PYTHON) scripts/ci_test.py
reports/adoption_drift_report.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-16T00:06:24Z",
"generated_at": "2026-06-16T00:18:35Z",
"skill_dir": ".",
"privacy_contract": {
"storage": "local-first",
reports/architecture_maintainability.json
+10 -17
View File
@@ -4,9 +4,9 @@
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"python_file_count": 191,
"python_file_count": 192,
"script_file_count": 122,
"test_file_count": 69,
"test_file_count": 70,
"internal_module_count": 39,
"cli_script_count": 85,
"command_handler_count": 68,
@@ -16,7 +16,7 @@
"watch_line_threshold": 720,
"block_line_threshold": 1500,
"largest_file_lines": 899,
"watchlist_count": 10,
"watchlist_count": 9,
"hotspot_count": 0,
"blocker_count": 0,
"decision": "pass"
@@ -29,13 +29,6 @@
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "tests/verify_world_class_evidence_intake.py",
"lines": 874,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/yao_cli_parser.py",
"lines": 870,
@@ -105,6 +98,13 @@
"kind": "cli-script",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
},
{
"path": "scripts/world_class_evidence_contract.py",
"lines": 686,
"kind": "internal-module",
"severity": "pass",
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
}
],
"watchlist": [
@@ -115,13 +115,6 @@
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "tests/verify_world_class_evidence_intake.py",
"lines": 874,
"kind": "test",
"severity": "pass",
"recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands."
},
{
"path": "scripts/yao_cli_parser.py",
"lines": 870,
reports/architecture_maintainability.md
+4 -5
View File
@@ -5,9 +5,9 @@ Generated at: `2026-06-13`
## Summary
- decision: `pass`
- python files: `191`
- python files: `192`
- scripts: `122`
- tests: `69`
- tests: `70`
- internal modules: `39`
- CLI scripts: `85`
- Yao CLI command handlers: `68`
@@ -15,7 +15,7 @@ Generated at: `2026-06-13`
- command modules: `6`
- largest file lines: `899`
- watch threshold lines: `720`
- watchlist: `10`
- watchlist: `9`
- hotspots: `0`
- blockers: `0`
@@ -30,7 +30,6 @@ No file-size hotspots found.
| File | Lines | Kind | Recommended next split |
| --- | ---: | --- | --- |
| `tests/verify_yao_cli.py` | `899` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
| `scripts/yao_cli_parser.py` | `870` | `internal-module` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
| `tests/verify_review_studio.py` | `836` | `test` | Break broad integration assertions into focused verifier helpers when the next behavior change lands. |
| `scripts/render_world_class_preflight.py` | `814` | `cli-script` | Watch this file before adding new responsibilities; extract a helper module when one concern dominates. |
@@ -45,7 +44,6 @@ No file-size hotspots found.
| File | Lines | Kind | Severity |
| --- | ---: | --- | --- |
| `tests/verify_yao_cli.py` | `899` | `test` | `pass` |
| `tests/verify_world_class_evidence_intake.py` | `874` | `test` | `pass` |
| `scripts/yao_cli_parser.py` | `870` | `internal-module` | `pass` |
| `tests/verify_review_studio.py` | `836` | `test` | `pass` |
| `scripts/render_world_class_preflight.py` | `814` | `cli-script` | `pass` |
@@ -56,6 +54,7 @@ No file-size hotspots found.
| `scripts/optimize_description.py` | `723` | `cli-script` | `pass` |
| `scripts/trust_check.py` | `714` | `internal-module` | `pass` |
| `scripts/apply_adaptation.py` | `706` | `cli-script` | `pass` |
| `scripts/world_class_evidence_contract.py` | `686` | `internal-module` | `pass` |
## Release Rule
reports/benchmark_reproducibility.json
+28 -14
View File
@@ -3,21 +3,34 @@
"ok": true,
"generated_at": "2026-06-16",
"skill_dir": ".",
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"git_status": {
"available": true,
"dirty": false,
"changed_file_count": 0,
"sample": [],
"dirty": true,
"changed_file_count": 22,
"sample": [
" M Makefile",
" M reports/benchmark_reproducibility.json",
" M reports/benchmark_reproducibility.md",
" M reports/context_budget.json",
" M reports/context_budget.md",
" M reports/context_budget_summary.json",
" M reports/evidence_consistency.json",
" M reports/output_execution_runs.json",
" M reports/output_execution_runs.md",
" M reports/review-studio.html",
" M reports/review-studio.json",
" M reports/review-viewer.json"
],
"scope": "generation-time status before this report is written"
},
"summary": {
"reproducibility_ready": true,
"release_lock_ready": true,
"release_lock_ready": false,
"methodology_complete": true,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a",
"evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5",
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
"output_case_count": 5,
@@ -37,14 +50,15 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4,
"working_tree_dirty": false,
"changed_file_count": 0
"public_claim_blocker_count": 5,
"working_tree_dirty": true,
"changed_file_count": 22
},
"public_claim": {
"ready": false,
"scope": "public benchmark or world-class readiness claim",
"blockers": [
"release lock is not clean or commit is unavailable",
"provider-backed model holdout evidence is incomplete",
"human blind-review adjudication is incomplete",
"world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)",
@@ -53,10 +67,10 @@
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks."
},
"release_lock": {
"ready": true,
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"ready": false,
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"status_scope": "generation-time status before this report is written",
"reason": "clean generation-time HEAD"
"reason": "working tree was dirty at generation time"
},
"evidence_bundle": {
"algorithm": "sha256(path,label,exists,artifact_sha256)",
@@ -64,7 +78,7 @@
"existing_count": 25,
"missing_count": 0,
"missing_paths": [],
"sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a"
"sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5"
},
"methodology": {
"path": "reports/benchmark_methodology.md",
@@ -138,7 +152,7 @@
"path": "reports/output_execution_runs.json",
"exists": true,
"bytes": 7967,
"sha256": "072e7ff6ef9c938f9b876bd950a69ede0c3d87939fc79f10c0b69b6df250c45d"
"sha256": "dd3e393370f82598b30f0d3de02191a02ff4b596a76463f4995a290517ebb15d"
},
{
"label": "blind_review",
reports/benchmark_reproducibility.md
+11 -10
View File
@@ -1,14 +1,14 @@
# Benchmark Reproducibility
Generated at: `2026-06-16`
Commit: `0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8`
Working tree dirty at generation: `false`
Evidence bundle SHA256: `51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a`
Commit: `e74ebe23595710a237e83de176d4fd2e46067824`
Working tree dirty at generation: `true`
Evidence bundle SHA256: `9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5`
## Summary
- reproducibility ready: `true`
- release lock ready: `true`
- release lock ready: `false`
- methodology complete: `true`
- required artifacts: `25`
- missing artifacts: `0`
@@ -22,8 +22,8 @@ Evidence bundle SHA256: `51261159180a2763fe1239729cb6e3da86bf76811560baa342f7420
- world-class ready: `false`
- world-class source checks: `7` pass / `13` total; `6` blocked
- public claim ready: `false`
- public claim blockers: `4`
- changed files at generation: `0`
- public claim blockers: `5`
- changed files at generation: `22`
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.
@@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi
| Blocker |
| --- |
| release lock is not clean or commit is unavailable |
| provider-backed model holdout evidence is incomplete |
| human blind-review adjudication is incomplete |
| world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) |
@@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi
## Release Lock
- ready: `true`
- reason: clean generation-time HEAD
- ready: `false`
- reason: working tree was dirty at generation time
- status scope: generation-time status before this report is written
## Evidence Bundle
- algorithm: `sha256(path,label,exists,artifact_sha256)`
- artifacts: `25` / `25`
- sha256: `51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a`
- sha256: `9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5`
## Methodology Sections
@@ -72,7 +73,7 @@ This report proves local benchmark reproducibility only. It keeps external provi
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
| output_execution | `reports/output_execution_runs.json` | present | `072e7ff6ef9c` |
| output_execution | `reports/output_execution_runs.json` | present | `dd3e393370f8` |
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
| review_adjudication | `reports/output_review_adjudication.json` | present | `bb8c72a9291e` |
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
reports/context_budget.json
+7 -7
View File
@@ -6,15 +6,15 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 797,
"other_text_tokens": 1046015,
"other_text_tokens": 1046308,
"estimated_initial_load_tokens": 990,
"estimated_total_text_tokens": 1046812,
"deferred_resource_tokens": 484786,
"estimated_total_text_tokens": 1047105,
"deferred_resource_tokens": 484796,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122
},
{
@@ -36,7 +36,7 @@
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122
}
],
@@ -59,14 +59,14 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
"summary": "Large deferred resources are indexed and backed by evidence."
},
"relevant_file_count": 630,
"relevant_file_count": 631,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 131.3
reports/context_budget.md
+1 -1
View File
@@ -2,7 +2,7 @@
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
| root | `.` | `production` | 1000 | 990 | 797 | 484786 | `governed` | scripts:424808 | 131.3 | - | ok |
| root | `.` | `production` | 1000 | 990 | 797 | 484796 | `governed` | scripts:424818 | 131.3 | - | ok |
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
reports/context_budget_summary.json
+4 -4
View File
@@ -1,5 +1,5 @@
{
"generated_at": "2026-06-16",
"generated_at": "2026-03-31",
"targets": [
{
"label": "root",
@@ -8,11 +8,11 @@
"budget_limit": 1000,
"initial_tokens": 990,
"skill_body_tokens": 797,
"deferred_resource_tokens": 484786,
"deferred_resource_tokens": 484796,
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122
}
],
@@ -35,7 +35,7 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
reports/evidence_consistency.json
+17 -17
View File
@@ -189,12 +189,12 @@
"status": "pass",
"expected": {
"status": "pass",
"detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3",
"detail": "initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3",
"evidence": "reports/context_budget.json"
},
"actual": {
"status": "pass",
"detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3",
"detail": "initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3",
"evidence": "reports/context_budget.json"
},
"paths": [
@@ -207,8 +207,8 @@
"key": "benchmark-release-lock-self-consistency",
"label": "Benchmark release lock matches git dirty state",
"status": "pass",
"expected": true,
"actual": true,
"expected": false,
"actual": false,
"paths": [
"reports/benchmark_reproducibility.json"
],
@@ -248,8 +248,8 @@
"key": "overview-benchmark-commit",
"label": "overview embeds the benchmark commit",
"status": "pass",
"expected": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"actual": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"expected": "e74ebe23595710a237e83de176d4fd2e46067824",
"actual": "e74ebe23595710a237e83de176d4fd2e46067824",
"paths": [
"reports/benchmark_reproducibility.json",
"reports/skill-overview.json"
@@ -261,7 +261,7 @@
"label": "overview embeds benchmark summary fields",
"status": "pass",
"expected": {
"release_lock_ready": true,
"release_lock_ready": false,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
@@ -271,10 +271,10 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4
"public_claim_blocker_count": 5
},
"actual": {
"release_lock_ready": true,
"release_lock_ready": false,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
@@ -284,7 +284,7 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4
"public_claim_blocker_count": 5
},
"paths": [
"reports/benchmark_reproducibility.json",
@@ -394,8 +394,8 @@
"key": "interpretation-benchmark-commit",
"label": "interpretation embeds the benchmark commit",
"status": "pass",
"expected": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"actual": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"expected": "e74ebe23595710a237e83de176d4fd2e46067824",
"actual": "e74ebe23595710a237e83de176d4fd2e46067824",
"paths": [
"reports/benchmark_reproducibility.json",
"reports/skill-interpretation.json"
@@ -407,7 +407,7 @@
"label": "interpretation embeds benchmark summary fields",
"status": "pass",
"expected": {
"release_lock_ready": true,
"release_lock_ready": false,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
@@ -417,10 +417,10 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4
"public_claim_blocker_count": 5
},
"actual": {
"release_lock_ready": true,
"release_lock_ready": false,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
@@ -430,7 +430,7 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4
"public_claim_blocker_count": 5
},
"paths": [
"reports/benchmark_reproducibility.json",
@@ -2411,7 +2411,7 @@
"`25` required artifacts",
"`23` reproduction commands",
"initial load `990/1000`",
"target count is `80`"
"target count is `81`"
],
"actual": "all present",
"paths": [
reports/install_simulation.json
+1 -1
View File
@@ -8,7 +8,7 @@
"installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill",
"summary": {
"archive_present": true,
"archive_entry_count": 647,
"archive_entry_count": 648,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 46.38,
"duration_ms": 42.36,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 45.36,
"duration_ms": 55.84,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 48.62,
"duration_ms": 43.71,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 43.77,
"duration_ms": 43.07,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 43.21,
"duration_ms": 42.02,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 40.08,
"duration_ms": 43.49,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 46.12,
"duration_ms": 40.43,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 43.65,
"duration_ms": 46.29,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 40.54,
"duration_ms": 42.67,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 40.25,
"duration_ms": 41.76,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 46.38 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 45.36 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 48.62 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 43.77 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 43.21 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 40.08 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 46.12 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 43.65 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 40.54 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 40.25 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 42.36 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 55.84 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 43.71 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 43.07 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 42.02 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 43.49 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 40.43 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 46.29 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 42.67 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 41.76 | 98 | 100.0 | pass |
## Next Fixes
reports/python_compatibility.json
+7 -1
View File
@@ -5,7 +5,7 @@
"root": ".",
"summary": {
"target_python": "3.11",
"file_count": 194,
"file_count": 195,
"issue_count": 0,
"syntax_error_count": 0,
"fstring_311_violation_count": 0,
@@ -1168,6 +1168,12 @@
"issue_count": 0,
"issues": []
},
{
"path": "tests/verify_world_class_submission_kit.py",
"ok": true,
"issue_count": 0,
"issues": []
},
{
"path": "tests/verify_world_class_submission_review.py",
"ok": true,
reports/python_compatibility.md
+1 -1
View File
@@ -6,7 +6,7 @@ Generated at: `2026-06-13`
- decision: `pass`
- target python: `3.11`
- files scanned: `194`
- files scanned: `195`
- issues: `0`
- syntax errors: `0`
- f-string 3.11 violations: `0`
reports/review-studio.html
+8 -8
View File
@@ -740,12 +740,12 @@
<section>
<h2>核心指标</h2>
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review Kit</span><strong>0/5</strong><p>pending 5; answer key hidden</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Public Claim</span><strong>blocked</strong><p>4 blockers; local reproducible true</p></article><article class='metric'><span>Blueprint</span><strong>21/21</strong><p>2.0 coverage; extensions partial 0, planned 0; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native; 4 installer-enforced</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>122 scripts scanned; secrets found</p></article><article class='metric'><span>Py Compat</span><strong>0</strong><p>194 files scanned for Python 3.11</p></article><article class='metric'><span>Arch Debt</span><strong>0</strong><p>899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Daily Ops</span><strong>5</strong><p>proposal-review; approval 0; release lock true</p></article><article class='metric'><span>Weekly Queue</span><strong>5</strong><p>curator-review; ready 1; top score 88</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Claim Guard</span><strong>0</strong><p>180 public surfaces scanned</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>647 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review Kit</span><strong>0/5</strong><p>pending 5; answer key hidden</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Public Claim</span><strong>blocked</strong><p>5 blockers; local reproducible true</p></article><article class='metric'><span>Blueprint</span><strong>21/21</strong><p>2.0 coverage; extensions partial 0, planned 0; evidence pending 4</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native; 4 installer-enforced</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>122 scripts scanned; secrets found</p></article><article class='metric'><span>Py Compat</span><strong>0</strong><p>194 files scanned for Python 3.11</p></article><article class='metric'><span>Arch Debt</span><strong>0</strong><p>899 largest lines; 10 watchlist; 68 CLI handlers; 18 in entrypoint</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Daily Ops</span><strong>5</strong><p>proposal-review; approval 0; release lock false</p></article><article class='metric'><span>Weekly Queue</span><strong>5</strong><p>curator-review; ready 1; top score 88</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Intake</span><strong>4/4</strong><p>0 valid submissions; 0 invalid</p></article><article class='metric'><span>Claim Guard</span><strong>0</strong><p>180 public surfaces scanned</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>647 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
</section>
<section>
<h2>审查闸门</h2>
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>Python 兼容</h3></div><p>Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards</p><footer>reports/python_compatibility.json <a href='python_compatibility.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>架构维护</h3></div><p>191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint</p><footer>reports/architecture_maintainability.json <a href='architecture_maintainability.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true</p><footer>reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>世界证据</h3></div><p>4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true</p><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 122 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>Python 兼容</h3></div><p>Python 3.11; 194 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards</p><footer>reports/python_compatibility.json <a href='python_compatibility.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>架构维护</h3></div><p>191 Python files; 0 hotspots; 10 watchlist files; 0 blockers; largest 899 lines; 68 CLI handlers; 18 in entrypoint</p><footer>reports/architecture_maintainability.json <a href='architecture_maintainability.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false</p><footer>reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>世界证据</h3></div><p>4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true</p><footer>reports/world_class_evidence_ledger.json <a href='world_class_evidence_ledger.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
</section>
<div class="twocol">
@@ -807,7 +807,7 @@
</section>
<section class="twocol">
<div class="panel"><h2>上下文</h2><p>initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3</p></div>
<div class="panel"><h2>上下文</h2><p>initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3</p></div>
<div class="panel"><h2>编译证据</h2><p>Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.</p></div>
</section>
@@ -837,14 +837,14 @@
</section>
<section id="telemetry" class="twocol">
<div class="panel"><h2>运营回路</h2><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true</p></div>
<div class="panel"><h2>运营回路</h2><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false</p></div>
<div class="panel"><h2>漂移信号</h2><dl class='kv-grid'><div><dt>事件数</dt><dd>1</dd></div><div><dt>采用率</dt><dd>100</dd></div><div><dt>漏触发</dt><dd>0</dd></div><div><dt>Bad Output Count</dt><dd>0</dd></div><div><dt>风险带</dt><dd>low</dd></div></dl></div>
</section>
<section class="twocol">
<div class="panel"><h2>日常运维</h2><dl class='kv-grid'><div><dt>决策</dt><dd>proposal-review</dd></div><div><dt>提案数</dt><dd>5</dd></div><div><dt>Approval Count</dt><dd>0</dd></div><div><dt>Pending Review Count</dt><dd>0</dd></div><div><dt>发布锁</dt><dd>是</dd></div><div><dt>世界级</dt><dd>否</dd></div><div><dt>Writes Source Files</dt><dd>否</dd></div><div><dt>Auto Patch Enabled</dt><dd>否</dd></div></dl></div>
<div class="panel"><h2>周度队列</h2><dl class='kv-grid'><div><dt>决策</dt><dd>curator-review</dd></div><div><dt>Week Id</dt><dd>2026-W25</dd></div><div><dt>日报数</dt><dd>1</dd></div><div><dt>唯一机会</dt><dd>5</dd></div><div><dt>待批准复核</dt><dd>1</dd></div><div><dt>提案复核</dt><dd>4</dd></div><div><dt>Top Score</dt><dd>88</dd></div><div><dt>发布锁</dt><dd>是</dd></div><div><dt>Writes Source Files</dt><dd>否</dd></div><div><dt>Auto Patch Enabled</dt><dd>否</dd></div></dl></div>
<div class="panel"><h2>日常运维</h2><dl class='kv-grid'><div><dt>决策</dt><dd>proposal-review</dd></div><div><dt>提案数</dt><dd>5</dd></div><div><dt>Approval Count</dt><dd>0</dd></div><div><dt>Pending Review Count</dt><dd>0</dd></div><div><dt>发布锁</dt><dd>否</dd></div><div><dt>世界级</dt><dd>否</dd></div><div><dt>Writes Source Files</dt><dd>否</dd></div><div><dt>Auto Patch Enabled</dt><dd>否</dd></div></dl></div>
<div class="panel"><h2>周度队列</h2><dl class='kv-grid'><div><dt>决策</dt><dd>curator-review</dd></div><div><dt>Week Id</dt><dd>2026-W25</dd></div><div><dt>日报数</dt><dd>1</dd></div><div><dt>唯一机会</dt><dd>5</dd></div><div><dt>待批准复核</dt><dd>1</dd></div><div><dt>提案复核</dt><dd>4</dd></div><div><dt>Top Score</dt><dd>88</dd></div><div><dt>发布锁</dt><dd>否</dd></div><div><dt>Writes Source Files</dt><dd>否</dd></div><div><dt>Auto Patch Enabled</dt><dd>否</dd></div></dl></div>
</section>
@@ -877,8 +877,8 @@
</section>
<section class="twocol">
<div class="panel"><h2>公开声明</h2><dl class='kv-grid'><div><dt>本地复现</dt><dd>是</dd></div><div><dt>发布锁</dt><dd>是</dd></div><div><dt>可公开声明</dt><dd>否</dd></div><div><dt>声明阻断</dt><dd>4</dd></div><div><dt>Provider 证据</dt><dd>否</dd></div><div><dt>人审完成</dt><dd>否</dd></div><div><dt>世界级就绪</dt><dd>否</dd></div></dl></div>
<div class="panel"><h2>声明阻断</h2><ul class='issues'><li><strong>阻断</strong><span>provider-backed model holdout evidence is incomplete</span></li><li><strong>阻断</strong><span>human blind-review adjudication is incomplete</span></li><li><strong>阻断</strong><span>world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)</span></li><li><strong>阻断</strong><span>world-class source checks are not all accepted (7/13 pass, 6 blocked)</span></li></ul></div>
<div class="panel"><h2>公开声明</h2><dl class='kv-grid'><div><dt>本地复现</dt><dd>是</dd></div><div><dt>发布锁</dt><dd>否</dd></div><div><dt>可公开声明</dt><dd>否</dd></div><div><dt>声明阻断</dt><dd>5</dd></div><div><dt>Provider 证据</dt><dd>否</dd></div><div><dt>人审完成</dt><dd>否</dd></div><div><dt>世界级就绪</dt><dd>否</dd></div></dl></div>
<div class="panel"><h2>声明阻断</h2><ul class='issues'><li><strong>阻断</strong><span>release lock is not clean or commit is unavailable</span></li><li><strong>阻断</strong><span>provider-backed model holdout evidence is incomplete</span></li><li><strong>阻断</strong><span>human blind-review adjudication is incomplete</span></li><li><strong>阻断</strong><span>world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)</span></li><li><strong>阻断</strong><span>world-class source checks are not all accepted (7/13 pass, 6 blocked)</span></li></ul></div>
</section>
<section class="twocol">
reports/review-studio.json
+59 -45
View File
@@ -43,7 +43,7 @@
"key": "context-budget",
"label": "上下文",
"status": "pass",
"detail": "initial load 990/1000; deferred 484786/120000; top deferred scripts 424808; resource governance governed; quality density 131.3",
"detail": "initial load 990/1000; deferred 484796/120000; top deferred scripts 424818; resource governance governed; quality density 131.3",
"evidence": "reports/context_budget.json",
"link": "context_budget.md"
},
@@ -107,7 +107,7 @@
"key": "operations-loop",
"label": "运营回路",
"status": "pass",
"detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true",
"detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false",
"evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly",
"link": "adoption_drift_report.md"
},
@@ -1913,11 +1913,11 @@
"ok": true,
"summary": {
"reproducibility_ready": true,
"release_lock_ready": true,
"release_lock_ready": false,
"methodology_complete": true,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a",
"evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5",
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
"output_case_count": 5,
@@ -1937,11 +1937,11 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4,
"working_tree_dirty": false,
"changed_file_count": 0
"public_claim_blocker_count": 5,
"working_tree_dirty": true,
"changed_file_count": 22
},
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"missing_artifacts": [],
"limitations": [
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
@@ -4593,7 +4593,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 46.38,
"duration_ms": 42.36,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4621,7 +4621,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 45.36,
"duration_ms": 55.84,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4644,7 +4644,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 48.62,
"duration_ms": 43.71,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4672,7 +4672,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 43.77,
"duration_ms": 43.07,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4695,7 +4695,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 43.21,
"duration_ms": 42.02,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4723,7 +4723,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 40.08,
"duration_ms": 43.49,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4746,7 +4746,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 46.12,
"duration_ms": 40.43,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4773,7 +4773,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 43.65,
"duration_ms": 46.29,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4796,7 +4796,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 40.54,
"duration_ms": 42.67,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -4825,7 +4825,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 40.25,
"duration_ms": 41.76,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -5551,21 +5551,34 @@
"ok": true,
"generated_at": "2026-06-16",
"skill_dir": ".",
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"git_status": {
"available": true,
"dirty": false,
"changed_file_count": 0,
"sample": [],
"dirty": true,
"changed_file_count": 22,
"sample": [
" M Makefile",
" M reports/benchmark_reproducibility.json",
" M reports/benchmark_reproducibility.md",
" M reports/context_budget.json",
" M reports/context_budget.md",
" M reports/context_budget_summary.json",
" M reports/evidence_consistency.json",
" M reports/output_execution_runs.json",
" M reports/output_execution_runs.md",
" M reports/review-studio.html",
" M reports/review-studio.json",
" M reports/review-viewer.json"
],
"scope": "generation-time status before this report is written"
},
"summary": {
"reproducibility_ready": true,
"release_lock_ready": true,
"release_lock_ready": false,
"methodology_complete": true,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a",
"evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5",
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
"output_case_count": 5,
@@ -5585,14 +5598,15 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4,
"working_tree_dirty": false,
"changed_file_count": 0
"public_claim_blocker_count": 5,
"working_tree_dirty": true,
"changed_file_count": 22
},
"public_claim": {
"ready": false,
"scope": "public benchmark or world-class readiness claim",
"blockers": [
"release lock is not clean or commit is unavailable",
"provider-backed model holdout evidence is incomplete",
"human blind-review adjudication is incomplete",
"world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)",
@@ -5601,10 +5615,10 @@
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks."
},
"release_lock": {
"ready": true,
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"ready": false,
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"status_scope": "generation-time status before this report is written",
"reason": "clean generation-time HEAD"
"reason": "working tree was dirty at generation time"
},
"evidence_bundle": {
"algorithm": "sha256(path,label,exists,artifact_sha256)",
@@ -5612,7 +5626,7 @@
"existing_count": 25,
"missing_count": 0,
"missing_paths": [],
"sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a"
"sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5"
},
"methodology": {
"path": "reports/benchmark_methodology.md",
@@ -5686,7 +5700,7 @@
"path": "reports/output_execution_runs.json",
"exists": true,
"bytes": 7967,
"sha256": "072e7ff6ef9c938f9b876bd950a69ede0c3d87939fc79f10c0b69b6df250c45d"
"sha256": "dd3e393370f82598b30f0d3de02191a02ff4b596a76463f4995a290517ebb15d"
},
{
"label": "blind_review",
@@ -16350,15 +16364,15 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 797,
"other_text_tokens": 1046015,
"other_text_tokens": 1046381,
"estimated_initial_load_tokens": 990,
"estimated_total_text_tokens": 1046812,
"deferred_resource_tokens": 484786,
"estimated_total_text_tokens": 1047178,
"deferred_resource_tokens": 484796,
"deferred_resource_warn_threshold": 120000,
"deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122
},
{
@@ -16380,7 +16394,7 @@
"large_deferred_resource_dirs": [
{
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122
}
],
@@ -16403,14 +16417,14 @@
],
"missing": [],
"path": "scripts",
"estimated_tokens": 424808,
"estimated_tokens": 424818,
"file_count": 122,
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
}
],
"summary": "Large deferred resources are indexed and backed by evidence."
},
"relevant_file_count": 630,
"relevant_file_count": 631,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 131.3
@@ -18394,7 +18408,7 @@
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"writes_source_files": false,
"auto_patch_enabled": false,
@@ -18411,7 +18425,7 @@
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"writes_source_files": false,
"auto_patch_enabled": false,
@@ -18860,7 +18874,7 @@
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true
},
"actions": [
@@ -18909,7 +18923,7 @@
"top_score": 88,
"skill_count": 12,
"actionable_portfolio_issue_count": 0,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
@@ -18929,7 +18943,7 @@
"top_score": 88,
"skill_count": 12,
"actionable_portfolio_issue_count": 0,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
@@ -19181,7 +19195,7 @@
"no_route_opportunity_count": 0
},
"release_state": {
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"public_world_class_ready": false,
"world_class_pending_count": 4
reports/review-viewer.json
+6 -6
View File
@@ -996,11 +996,11 @@
"ok": true,
"summary": {
"reproducibility_ready": true,
"release_lock_ready": true,
"release_lock_ready": false,
"methodology_complete": true,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a",
"evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5",
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
"output_case_count": 5,
@@ -1020,11 +1020,11 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4,
"working_tree_dirty": false,
"changed_file_count": 0
"public_claim_blocker_count": 5,
"working_tree_dirty": true,
"changed_file_count": 22
},
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"missing_artifacts": [],
"limitations": [
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 213,
"package_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f"
"package_sha256": "5599cd15154d9fa948e9907f5ada8e22486166be102739eed26e0a1fdad3a326"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `213`
- Package SHA256: `a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f`
- Package SHA256: `5599cd15154d9fa948e9907f5ada8e22486166be102739eed26e0a1fdad3a326`
## Failures
reports/skill-interpretation.json
+6 -6
View File
@@ -1000,11 +1000,11 @@
"ok": true,
"summary": {
"reproducibility_ready": true,
"release_lock_ready": true,
"release_lock_ready": false,
"methodology_complete": true,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a",
"evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5",
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
"output_case_count": 5,
@@ -1024,11 +1024,11 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4,
"working_tree_dirty": false,
"changed_file_count": 0
"public_claim_blocker_count": 5,
"working_tree_dirty": true,
"changed_file_count": 22
},
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"missing_artifacts": [],
"limitations": [
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
reports/skill-os-2-review.md
+3 -3
View File
@@ -66,8 +66,8 @@ This is still not the final world-class state. Target-native behavior contracts
| Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed |
| World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed |
| World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed |
| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename and nested raw-field rejection | v0 landed |
| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_evidence_intake.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed |
| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, and nested raw-field rejection | v0 landed |
| World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed |
| Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed |
| Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed |
| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed |
@@ -146,7 +146,7 @@ Next move: add real client or installer permission enforcement integration.
| Benchmark Reproducibility | local reproducibility ready with `25` required artifacts, `0` missing artifacts, `23` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations |
| IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable |
| Context Budget | initial load `990/1000`, under the production budget |
| CI | `make ci-test` target count is `80` after the Weekly SkillOps Curator gate update |
| CI | `make ci-test` target count is `81` after the World-Class Submission Kit verifier split |
## Next Highest-Leverage Moves
reports/skill-overview.json
+6 -6
View File
@@ -995,11 +995,11 @@
"ok": true,
"summary": {
"reproducibility_ready": true,
"release_lock_ready": true,
"release_lock_ready": false,
"methodology_complete": true,
"required_artifact_count": 25,
"missing_artifact_count": 0,
"evidence_bundle_sha256": "51261159180a2763fe1239729cb6e3da86bf76811560baa342f742086d0c3c6a",
"evidence_bundle_sha256": "9878e207b4683df3b85fadf4052d933d2a1953c7ced03bfb4e35626b940d7fa5",
"source_contract_sha256": "a5f307ba8e89acb55d03b1bcee3e0ea50471af388511f2e53c73aff6d96ffb7f",
"archive_sha256": "996e36863203c30d3c518e0d635be054177dc5b774c9cd3d8311f77d69013a81",
"output_case_count": 5,
@@ -1019,11 +1019,11 @@
"world_class_source_pass_count": 7,
"world_class_source_blocked_count": 6,
"public_claim_ready": false,
"public_claim_blocker_count": 4,
"working_tree_dirty": false,
"changed_file_count": 0
"public_claim_blocker_count": 5,
"working_tree_dirty": true,
"changed_file_count": 22
},
"commit": "0ee1f89637ac1c26d70498e6d726fcffbbc5e8e8",
"commit": "e74ebe23595710a237e83de176d4fd2e46067824",
"missing_artifacts": [],
"limitations": [
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
reports/skill_os2_audit.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"item_count": 15,
reports/skill_os2_audit.md
+1 -1
View File
@@ -1,6 +1,6 @@
# Skill OS 2.0 Audit
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/skill_os2_coverage.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"item_count": 21,
reports/skill_os2_coverage.md
+1 -1
View File
@@ -1,6 +1,6 @@
# Skill OS 2.0 Blueprint Coverage
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/skillops/daily/2026-06-16.json
+3 -3
View File
@@ -14,7 +14,7 @@
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"writes_source_files": false,
"auto_patch_enabled": false,
@@ -31,7 +31,7 @@
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"writes_source_files": false,
"auto_patch_enabled": false,
@@ -480,7 +480,7 @@
"local_blueprint_ready": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true
},
"actions": [
reports/skillops/daily/2026-06-16.md
+1 -1
View File
@@ -14,7 +14,7 @@ Generated at: `2026-06-16`
- local blueprint ready: `true`
- public world-class ready: `false`
- world-class pending: `4`
- release lock ready: `true`
- release lock ready: `false`
- evidence consistency ok: `true`
This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence.
reports/skillops/weekly/2026-W25.json
+3 -3
View File
@@ -15,7 +15,7 @@
"top_score": 88,
"skill_count": 12,
"actionable_portfolio_issue_count": 0,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
@@ -35,7 +35,7 @@
"top_score": 88,
"skill_count": 12,
"actionable_portfolio_issue_count": 0,
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"public_world_class_ready": false,
"world_class_pending_count": 4,
@@ -287,7 +287,7 @@
"no_route_opportunity_count": 0
},
"release_state": {
"release_lock_ready": true,
"release_lock_ready": false,
"evidence_consistency_ok": true,
"public_world_class_ready": false,
"world_class_pending_count": 4
reports/skillops/weekly/2026-W25.md
+1 -1
View File
@@ -13,7 +13,7 @@ Week: `2026-W25`
- top_score: `88`
- skill_count: `12`
- actionable_portfolio_issue_count: `0`
- release_lock_ready: `True`
- release_lock_ready: `False`
- evidence_consistency_ok: `True`
- public_world_class_ready: `False`
- world_class_pending_count: `4`
reports/world_class_claim_guard.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"ledger_ready_to_claim_world_class": false,
reports/world_class_claim_guard.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Claim Guard
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/world_class_evidence_intake.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"schema_present": true,
reports/world_class_evidence_intake.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Evidence Intake
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/world_class_evidence_ledger.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"decision": "evidence-pending",
"ready_to_claim_world_class": false,
reports/world_class_evidence_ledger.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Evidence Ledger
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/world_class_evidence_plan.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"audit_decision": "continue-iteration",
reports/world_class_evidence_plan.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Evidence Plan
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/world_class_evidence_preflight.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"evidence_item_count": 4,
reports/world_class_evidence_preflight.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Evidence Preflight
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/world_class_operator_runbook.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"evidence_item_count": 4,
reports/world_class_operator_runbook.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Operator Runbook
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
reports/world_class_submission_review.json
+1 -1
View File
@@ -1,7 +1,7 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-16",
"generated_at": "2026-06-13",
"skill_dir": ".",
"decision": "awaiting-submissions",
"ready_to_claim_world_class": false,
reports/world_class_submission_review.md
+1 -1
View File
@@ -1,6 +1,6 @@
# World-Class Submission Review
Generated at: `2026-06-16`
Generated at: `2026-06-13`
## Summary
scripts/ci_test.py
+1
View File
@@ -59,6 +59,7 @@ DEFAULT_TARGETS = [
"world-class-evidence-check",
"world-class-ledger-check",
"world-class-intake-check",
"world-class-submission-kit-check",
"world-class-preflight-check",
"world-class-submission-review-check",
"world-class-runbook-check",
tests/verify_world_class_evidence_intake.py
+6 -220
View File
@@ -9,7 +9,6 @@ from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py"
KIT_SCRIPT = ROOT / "scripts" / "prepare_world_class_submission_kit.py"
TMP = ROOT / "tests" / "tmp_world_class_evidence_intake"
sys.path.insert(0, str(ROOT / "scripts"))
@@ -25,6 +24,11 @@ def sha256_file(path: Path) -> str:
def run_intake(*extra: str) -> dict:
extra_args = list(extra)
if "--output-json" not in extra_args:
extra_args.extend(["--output-json", str(TMP / "last_intake.json")])
if "--output-md" not in extra_args:
extra_args.extend(["--output-md", str(TMP / "last_intake.md")])
proc = subprocess.run(
[
sys.executable,
@@ -32,7 +36,7 @@ def run_intake(*extra: str) -> dict:
str(ROOT),
"--generated-at",
"2026-06-14",
*extra,
*extra_args,
],
cwd=ROOT,
capture_output=True,
@@ -42,23 +46,6 @@ def run_intake(*extra: str) -> dict:
return json.loads(proc.stdout)
def run_kit(*extra: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
sys.executable,
str(KIT_SCRIPT),
str(ROOT),
"--generated-at",
"2026-06-14",
*extra,
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
def provider_submission(*, valid: bool, artifact_path: str = "reports/output_execution_runs.json") -> dict:
return {
"schema_version": "1.0",
@@ -516,27 +503,9 @@ def assert_external_contract_artifact_validation() -> None:
assert any("summary.adoption_sample_count must be >0" in error for error in telemetry_invalid["errors"]), telemetry_invalid["errors"]
def assert_documented_submission_commands() -> None:
expected_fragments = [
'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"',
'world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR"',
'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR"',
'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" --prefill-artifacts',
'world-class-intake . --submissions-dir "$SUBMISSIONS_DIR"',
'world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR"',
'world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR"',
]
for relative_path in ("README.md", "evidence/world_class/README.md"):
text = (ROOT / relative_path).read_text(encoding="utf-8")
for fragment in expected_fragments:
assert fragment in text, (relative_path, fragment)
assert "/tmp/yao-world-class-submission-kit" not in text, relative_path
def main() -> None:
shutil.rmtree(TMP, ignore_errors=True)
TMP.mkdir(parents=True, exist_ok=True)
assert_documented_submission_commands()
default_json = TMP / "world_class_evidence_intake.json"
default_md = TMP / "world_class_evidence_intake.md"
assert_human_contract_artifact_validation()
@@ -610,189 +579,6 @@ def main() -> None:
assert "Real submissions must include the evidence-key critical artifact paths with verified SHA-256 digests." in markdown, markdown
assert "Real submissions must replace template submitter, date, and provenance placeholders with concrete evidence metadata." in markdown, markdown
kit_dir = TMP / "submission_kit"
kit_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout")
kit_payload = json.loads(kit_proc.stdout)
assert kit_payload["ok"] is True, kit_payload
assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"]
assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["submission_ref_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["submission_ref_ready_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["supporting_artifact_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["supporting_artifact_ready_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"]
assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_collect_source_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_submission_ref_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_submission_ref_ready_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_supporting_artifact_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_supporting_artifact_ready_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"]
assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"]
assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"]
assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"]
assert kit_payload["files"][0]["output_path"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/provider-holdout.json"), kit_payload["files"]
assert len(kit_payload["evidence_matrix"]) == 1, kit_payload["evidence_matrix"]
matrix_row = kit_payload["evidence_matrix"][0]
assert matrix_row["evidence_key"] == "provider-holdout", matrix_row
assert matrix_row["stage"] == "collect-source", matrix_row
assert matrix_row["draft_status"] == "written", matrix_row
assert matrix_row["artifact_ready_count"] >= 1, matrix_row
assert matrix_row["artifact_total_count"] >= matrix_row["artifact_ready_count"], matrix_row
assert matrix_row["submission_ref_ready_count"] == 1, matrix_row
assert matrix_row["submission_ref_total_count"] == 1, matrix_row
assert matrix_row["supporting_artifact_ready_count"] >= 1, matrix_row
assert matrix_row["supporting_artifact_total_count"] >= matrix_row["supporting_artifact_ready_count"], matrix_row
assert matrix_row["source_pass_count"] == 1, matrix_row
assert matrix_row["source_check_count"] == 3, matrix_row
assert matrix_row["source_blocked_count"] == 2, matrix_row
assert matrix_row["counts_as_completion"] is False, matrix_row
assert "real credentials" in matrix_row["next_action"], matrix_row
artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]}
assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["artifact_role"] == "submission-ref", artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["submission_ref_required"] is True, artifact_rows
assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows
assert artifact_rows["reports/output_execution_runs.md"]["artifact_role"] == "supporting-evidence", artifact_rows
assert artifact_rows["reports/output_execution_runs.md"]["submission_ref_required"] is False, artifact_rows
source_rows = {item["field"]: item for item in kit_payload["source_checklist"]}
assert source_rows["model_executed_count"]["status"] == "blocked", source_rows
assert source_rows["model_executed_count"]["actual"] == 0, source_rows
assert source_rows["timing_observed_count"]["status"] == "pass", source_rows
assert source_rows["token_observed_count"]["status"] == "blocked", source_rows
kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8"))
assert kit_draft["template_only"] is True, kit_draft
assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft
assert "sha256" not in kit_draft["artifact_refs"][0], kit_draft
assert kit_draft["provenance"]["run_command"] == "python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60", kit_draft
assert kit_draft["provenance"]["credential_env"] == "OPENAI_API_KEY", kit_draft
assert "<redacted>" not in json.dumps(kit_draft), kit_draft
kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8"))
assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"]
assert kit_manifest["summary"]["artifact_prefill_enabled"] is False, kit_manifest["summary"]
assert kit_manifest["summary"]["artifact_ref_prefill_count"] == 0, kit_manifest["summary"]
assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"]
assert kit_manifest["source_checklist"] == kit_payload["source_checklist"], kit_manifest["source_checklist"]
assert kit_manifest["artifacts"]["html"].endswith("tests/tmp_world_class_evidence_intake/submission_kit/index.html"), kit_manifest["artifacts"]
kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8")
assert "Drafts are not accepted evidence." in kit_readme, kit_readme
assert "Execution Runbook" in kit_readme, kit_readme
assert "output-exec --provider-runner openai" in kit_readme, kit_readme
assert "<redacted>" not in kit_readme, kit_readme
assert "validate intake" in kit_readme, kit_readme
assert "Artifact Checklist" in kit_readme, kit_readme
assert "Evidence Matrix" in kit_readme, kit_readme
assert "Submission refs" in kit_readme, kit_readme
assert "Supporting assets" in kit_readme, kit_readme
assert "`submission-ref`" in kit_readme, kit_readme
assert "`supporting-evidence`" in kit_readme, kit_readme
assert "`collect-source`" in kit_readme, kit_readme
assert "Matrix rows are guidance only" in kit_readme, kit_readme
assert "Source Evidence Snapshot" in kit_readme, kit_readme
assert "reports/output_execution_runs.json" in kit_readme, kit_readme
assert "Provider model run" in kit_readme, kit_readme
kit_html = (kit_dir / "index.html").read_text(encoding="utf-8")
assert "<title>World-Class Evidence Submission Kit</title>" in kit_html, kit_html
assert "Drafts are not accepted evidence" in kit_html, kit_html
assert "provider-holdout" in kit_html, kit_html
assert "Artifact Checklist" in kit_html, kit_html
assert "Evidence Matrix" in kit_html, kit_html
assert "matrix-card collect-source" in kit_html, kit_html
assert (
f"<dt>Submission refs</dt><dd>{matrix_row['submission_ref_ready_count']}/{matrix_row['submission_ref_total_count']} ready</dd>"
in kit_html
), kit_html
assert (
f"<dt>Supporting assets</dt><dd>{matrix_row['supporting_artifact_ready_count']}/{matrix_row['supporting_artifact_total_count']} ready</dd>"
in kit_html
), kit_html
assert "<dt>Source</dt><dd>1/3 pass</dd>" in kit_html, kit_html
assert "Source Evidence Snapshot" in kit_html, kit_html
assert "<dt>Field</dt><dd><code>model_executed_count</code></dd>" in kit_html, kit_html
assert "<dt>Current</dt><dd><code>0</code></dd>" in kit_html, kit_html
assert "<dt>Expected</dt><dd><code>&gt;0</code></dd>" in kit_html, kit_html
assert "World-Class Evidence Submission Kit" in kit_html, kit_html
assert "Execution Runbook" in kit_html, kit_html
assert "output-exec --provider-runner openai" in kit_html, kit_html
assert "&lt;redacted&gt;" not in kit_html and "<redacted>" not in kit_html, kit_html
assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html
assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html
prefilled_kit_dir = TMP / "prefilled_submission_kit"
prefilled_proc = run_kit(
"--output-dir",
str(prefilled_kit_dir),
"--evidence-key",
"provider-holdout",
"--prefill-artifacts",
)
prefilled_payload = json.loads(prefilled_proc.stdout)
assert prefilled_payload["summary"]["artifact_prefill_enabled"] is True, prefilled_payload["summary"]
assert prefilled_payload["summary"]["artifact_ref_prefill_count"] == 1, prefilled_payload["summary"]
assert prefilled_payload["summary"]["artifact_ref_unfilled_count"] == 0, prefilled_payload["summary"]
assert prefilled_payload["summary"]["drafts_count_as_evidence"] is False, prefilled_payload["summary"]
assert prefilled_payload["safety"]["artifact_prefill_counts_as_evidence"] is False, prefilled_payload["safety"]
assert prefilled_payload["files"][0]["prefilled_artifact_ref_count"] == 1, prefilled_payload["files"]
prefilled_draft = json.loads((prefilled_kit_dir / "provider-holdout.json").read_text(encoding="utf-8"))
assert prefilled_draft["template_only"] is True, prefilled_draft
assert prefilled_draft["attestation"]["real_external_or_human_evidence"] is False, prefilled_draft
assert len(prefilled_draft["artifact_refs"][0]["sha256"]) == 64, prefilled_draft
assert prefilled_draft["artifact_refs"][0]["contains_raw_content"] is False, prefilled_draft
prefilled_readme = (prefilled_kit_dir / "README.md").read_text(encoding="utf-8")
assert "Optional artifact prefill only inserts SHA-256 digests" in prefilled_readme, prefilled_readme
assert "does not mark a draft as real evidence" in prefilled_readme, prefilled_readme
prefilled_html = (prefilled_kit_dir / "index.html").read_text(encoding="utf-8")
assert "artifact prefill only inserts local SHA-256 digests" in prefilled_html, prefilled_html
assert "<dt>Prefill</dt><dd>1 artifact refs</dd>" in prefilled_html, prefilled_html
native_kit_dir = TMP / "native_permission_kit"
native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement")
native_kit_payload = json.loads(native_kit_proc.stdout)
native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]}
assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 1, native_kit_payload["summary"]
if "dist/targets/openai/adapter.json" in native_rows:
assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows
assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows
assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows
else:
glob_rows = [
item for item in native_kit_payload["artifact_checklist"]
if item["source_pattern"] == "dist/targets/*/adapter.json"
]
assert len(glob_rows) == 1, glob_rows
assert glob_rows[0]["status"] == "glob-no-match", glob_rows[0]
assert glob_rows[0]["artifact_ref_ready"] is False, glob_rows[0]
assert glob_rows[0]["concrete_reference_required"] is True, glob_rows[0]
native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8")
assert "Glob patterns are expanded into concrete files" in native_readme, native_readme
draft_intake = run_intake("--submissions-dir", str(kit_dir))
assert draft_intake["ok"] is False, draft_intake
assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"]
assert draft_intake["summary"]["invalid_submission_count"] == 1, draft_intake["summary"]
assert draft_intake["submissions"][0]["evidence_key"] == "provider-holdout", draft_intake["submissions"]
assert all(item["evidence_key"] != "unknown" for item in draft_intake["submissions"]), draft_intake["submissions"]
spaced_kit_dir = TMP / "submission kit spaced"
spaced_kit_proc = run_kit("--output-dir", str(spaced_kit_dir), "--evidence-key", "provider-holdout")
spaced_kit_payload = json.loads(spaced_kit_proc.stdout)
quoted_spaced_dir = "'tests/tmp_world_class_evidence_intake/submission kit spaced'"
assert quoted_spaced_dir in spaced_kit_payload["commands"]["validate_intake"], spaced_kit_payload["commands"]
assert quoted_spaced_dir in spaced_kit_payload["commands"]["refresh_ledger"], spaced_kit_payload["commands"]
spaced_readme = (spaced_kit_dir / "README.md").read_text(encoding="utf-8")
assert quoted_spaced_dir in spaced_readme, spaced_readme
existing_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout")
existing_payload = json.loads(existing_proc.stdout)
assert existing_payload["summary"]["existing_count"] == 1, existing_payload["summary"]
assert existing_payload["files"][0]["status"] == "exists", existing_payload["files"]
valid_dir = TMP / "valid_submissions"
valid_dir.mkdir()
(valid_dir / "provider-holdout.json").write_text(
tests/verify_world_class_submission_kit.py
+277
View File
@@ -0,0 +1,277 @@
#!/usr/bin/env python3
import json
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
INTAKE_SCRIPT = ROOT / "scripts" / "render_world_class_evidence_intake.py"
KIT_SCRIPT = ROOT / "scripts" / "prepare_world_class_submission_kit.py"
TMP = ROOT / "tests" / "tmp_world_class_submission_kit"
def run_intake(*extra: str) -> dict:
extra_args = list(extra)
if "--output-json" not in extra_args:
extra_args.extend(["--output-json", str(TMP / "last_intake.json")])
if "--output-md" not in extra_args:
extra_args.extend(["--output-md", str(TMP / "last_intake.md")])
proc = subprocess.run(
[
sys.executable,
str(INTAKE_SCRIPT),
str(ROOT),
"--generated-at",
"2026-06-14",
*extra_args,
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
return json.loads(proc.stdout)
def run_kit(*extra: str) -> subprocess.CompletedProcess[str]:
return subprocess.run(
[
sys.executable,
str(KIT_SCRIPT),
str(ROOT),
"--generated-at",
"2026-06-14",
*extra,
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
def assert_documented_submission_commands() -> None:
expected_fragments = [
'SUBMISSIONS_DIR="${SUBMISSIONS_DIR:-evidence/world_class/submissions}"',
'world-class-preflight . --submissions-dir "$SUBMISSIONS_DIR"',
'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR"',
'world-class-submission-kit . --output-dir "$SUBMISSIONS_DIR" --prefill-artifacts',
'world-class-intake . --submissions-dir "$SUBMISSIONS_DIR"',
'world-class-submission-review . --submissions-dir "$SUBMISSIONS_DIR"',
'world-class-ledger . --submissions-dir "$SUBMISSIONS_DIR"',
]
for relative_path in ("README.md", "evidence/world_class/README.md"):
text = (ROOT / relative_path).read_text(encoding="utf-8")
for fragment in expected_fragments:
assert fragment in text, (relative_path, fragment)
assert "/tmp/yao-world-class-submission-kit" not in text, relative_path
def main() -> None:
shutil.rmtree(TMP, ignore_errors=True)
TMP.mkdir(parents=True, exist_ok=True)
assert_documented_submission_commands()
kit_dir = TMP / "submission_kit"
kit_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout")
kit_payload = json.loads(kit_proc.stdout)
assert kit_payload["ok"] is True, kit_payload
assert kit_payload["summary"]["decision"] == "submission-kit-ready", kit_payload["summary"]
assert kit_payload["summary"]["requested_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["written_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["artifact_checklist_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["artifact_ready_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["submission_ref_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["submission_ref_ready_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["supporting_artifact_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["supporting_artifact_ready_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["source_check_count"] == 3, kit_payload["summary"]
assert kit_payload["summary"]["source_pass_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["source_blocked_count"] == 2, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_collect_source_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_submission_ref_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_submission_ref_ready_count"] == 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_supporting_artifact_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_supporting_artifact_ready_count"] >= 1, kit_payload["summary"]
assert kit_payload["summary"]["evidence_matrix_counts_as_completion"] == 0, kit_payload["summary"]
assert kit_payload["summary"]["drafts_count_as_evidence"] is False, kit_payload["summary"]
assert kit_payload["safety"]["template_only_drafts"] is True, kit_payload["safety"]
assert kit_payload["safety"]["raw_content_allowed"] is False, kit_payload["safety"]
assert kit_payload["files"][0]["output_path"].endswith(
f"tests/{TMP.name}/submission_kit/provider-holdout.json"
), kit_payload["files"]
assert len(kit_payload["evidence_matrix"]) == 1, kit_payload["evidence_matrix"]
matrix_row = kit_payload["evidence_matrix"][0]
assert matrix_row["evidence_key"] == "provider-holdout", matrix_row
assert matrix_row["stage"] == "collect-source", matrix_row
assert matrix_row["draft_status"] == "written", matrix_row
assert matrix_row["artifact_ready_count"] >= 1, matrix_row
assert matrix_row["artifact_total_count"] >= matrix_row["artifact_ready_count"], matrix_row
assert matrix_row["submission_ref_ready_count"] == 1, matrix_row
assert matrix_row["submission_ref_total_count"] == 1, matrix_row
assert matrix_row["supporting_artifact_ready_count"] >= 1, matrix_row
assert matrix_row["supporting_artifact_total_count"] >= matrix_row["supporting_artifact_ready_count"], matrix_row
assert matrix_row["source_pass_count"] == 1, matrix_row
assert matrix_row["source_check_count"] == 3, matrix_row
assert matrix_row["source_blocked_count"] == 2, matrix_row
assert matrix_row["counts_as_completion"] is False, matrix_row
assert "real credentials" in matrix_row["next_action"], matrix_row
artifact_rows = {item["path"]: item for item in kit_payload["artifact_checklist"]}
assert "reports/output_execution_runs.json" in artifact_rows, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["artifact_ref_ready"] is True, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["artifact_role"] == "submission-ref", artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["submission_ref_required"] is True, artifact_rows
assert len(artifact_rows["reports/output_execution_runs.json"]["sha256"]) == 64, artifact_rows
assert artifact_rows["reports/output_execution_runs.json"]["contains_raw_content"] is False, artifact_rows
assert artifact_rows["reports/output_execution_runs.md"]["artifact_role"] == "supporting-evidence", artifact_rows
assert artifact_rows["reports/output_execution_runs.md"]["submission_ref_required"] is False, artifact_rows
source_rows = {item["field"]: item for item in kit_payload["source_checklist"]}
assert source_rows["model_executed_count"]["status"] == "blocked", source_rows
assert source_rows["model_executed_count"]["actual"] == 0, source_rows
assert source_rows["timing_observed_count"]["status"] == "pass", source_rows
assert source_rows["token_observed_count"]["status"] == "blocked", source_rows
kit_draft = json.loads((kit_dir / "provider-holdout.json").read_text(encoding="utf-8"))
assert kit_draft["template_only"] is True, kit_draft
assert kit_draft["attestation"]["real_external_or_human_evidence"] is False, kit_draft
assert "sha256" not in kit_draft["artifact_refs"][0], kit_draft
assert kit_draft["provenance"]["run_command"] == (
"python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60"
), kit_draft
assert kit_draft["provenance"]["credential_env"] == "OPENAI_API_KEY", kit_draft
assert "<redacted>" not in json.dumps(kit_draft), kit_draft
kit_manifest = json.loads((kit_dir / "submission_manifest.json").read_text(encoding="utf-8"))
assert kit_manifest["summary"]["ledger_counts_submission_as_completion"] is False, kit_manifest["summary"]
assert kit_manifest["summary"]["artifact_prefill_enabled"] is False, kit_manifest["summary"]
assert kit_manifest["summary"]["artifact_ref_prefill_count"] == 0, kit_manifest["summary"]
assert kit_manifest["artifact_checklist"] == kit_payload["artifact_checklist"], kit_manifest["artifact_checklist"]
assert kit_manifest["source_checklist"] == kit_payload["source_checklist"], kit_manifest["source_checklist"]
assert kit_manifest["artifacts"]["html"].endswith(
f"tests/{TMP.name}/submission_kit/index.html"
), kit_manifest["artifacts"]
kit_readme = (kit_dir / "README.md").read_text(encoding="utf-8")
assert "Drafts are not accepted evidence." in kit_readme, kit_readme
assert "Execution Runbook" in kit_readme, kit_readme
assert "output-exec --provider-runner openai" in kit_readme, kit_readme
assert "<redacted>" not in kit_readme, kit_readme
assert "validate intake" in kit_readme, kit_readme
assert "Artifact Checklist" in kit_readme, kit_readme
assert "Evidence Matrix" in kit_readme, kit_readme
assert "Submission refs" in kit_readme, kit_readme
assert "Supporting assets" in kit_readme, kit_readme
assert "`submission-ref`" in kit_readme, kit_readme
assert "`supporting-evidence`" in kit_readme, kit_readme
assert "`collect-source`" in kit_readme, kit_readme
assert "Matrix rows are guidance only" in kit_readme, kit_readme
assert "Source Evidence Snapshot" in kit_readme, kit_readme
assert "reports/output_execution_runs.json" in kit_readme, kit_readme
assert "Provider model run" in kit_readme, kit_readme
kit_html = (kit_dir / "index.html").read_text(encoding="utf-8")
assert "<title>World-Class Evidence Submission Kit</title>" in kit_html, kit_html
assert "Drafts are not accepted evidence" in kit_html, kit_html
assert "provider-holdout" in kit_html, kit_html
assert "Artifact Checklist" in kit_html, kit_html
assert "Evidence Matrix" in kit_html, kit_html
assert "matrix-card collect-source" in kit_html, kit_html
assert (
f"<dt>Submission refs</dt><dd>{matrix_row['submission_ref_ready_count']}/{matrix_row['submission_ref_total_count']} ready</dd>"
in kit_html
), kit_html
assert (
f"<dt>Supporting assets</dt><dd>{matrix_row['supporting_artifact_ready_count']}/{matrix_row['supporting_artifact_total_count']} ready</dd>"
in kit_html
), kit_html
assert "<dt>Source</dt><dd>1/3 pass</dd>" in kit_html, kit_html
assert "Source Evidence Snapshot" in kit_html, kit_html
assert "<dt>Field</dt><dd><code>model_executed_count</code></dd>" in kit_html, kit_html
assert "<dt>Current</dt><dd><code>0</code></dd>" in kit_html, kit_html
assert "<dt>Expected</dt><dd><code>&gt;0</code></dd>" in kit_html, kit_html
assert "World-Class Evidence Submission Kit" in kit_html, kit_html
assert "Execution Runbook" in kit_html, kit_html
assert "output-exec --provider-runner openai" in kit_html, kit_html
assert "&lt;redacted&gt;" not in kit_html and "<redacted>" not in kit_html, kit_html
assert "Do not include credentials, raw prompts, raw outputs, transcripts, notes, or private user content." in kit_html, kit_html
assert "Rows marked submission-ref are the paths expected in artifact_refs" in kit_html, kit_html
prefilled_kit_dir = TMP / "prefilled_submission_kit"
prefilled_proc = run_kit(
"--output-dir",
str(prefilled_kit_dir),
"--evidence-key",
"provider-holdout",
"--prefill-artifacts",
)
prefilled_payload = json.loads(prefilled_proc.stdout)
assert prefilled_payload["summary"]["artifact_prefill_enabled"] is True, prefilled_payload["summary"]
assert prefilled_payload["summary"]["artifact_ref_prefill_count"] == 1, prefilled_payload["summary"]
assert prefilled_payload["summary"]["artifact_ref_unfilled_count"] == 0, prefilled_payload["summary"]
assert prefilled_payload["summary"]["drafts_count_as_evidence"] is False, prefilled_payload["summary"]
assert prefilled_payload["safety"]["artifact_prefill_counts_as_evidence"] is False, prefilled_payload["safety"]
assert prefilled_payload["files"][0]["prefilled_artifact_ref_count"] == 1, prefilled_payload["files"]
prefilled_draft = json.loads((prefilled_kit_dir / "provider-holdout.json").read_text(encoding="utf-8"))
assert prefilled_draft["template_only"] is True, prefilled_draft
assert prefilled_draft["attestation"]["real_external_or_human_evidence"] is False, prefilled_draft
assert len(prefilled_draft["artifact_refs"][0]["sha256"]) == 64, prefilled_draft
assert prefilled_draft["artifact_refs"][0]["contains_raw_content"] is False, prefilled_draft
prefilled_readme = (prefilled_kit_dir / "README.md").read_text(encoding="utf-8")
assert "Optional artifact prefill only inserts SHA-256 digests" in prefilled_readme, prefilled_readme
assert "does not mark a draft as real evidence" in prefilled_readme, prefilled_readme
prefilled_html = (prefilled_kit_dir / "index.html").read_text(encoding="utf-8")
assert "artifact prefill only inserts local SHA-256 digests" in prefilled_html, prefilled_html
assert "<dt>Prefill</dt><dd>1 artifact refs</dd>" in prefilled_html, prefilled_html
native_kit_dir = TMP / "native_permission_kit"
native_kit_proc = run_kit("--output-dir", str(native_kit_dir), "--evidence-key", "native-permission-enforcement")
native_kit_payload = json.loads(native_kit_proc.stdout)
native_rows = {item["path"]: item for item in native_kit_payload["artifact_checklist"]}
assert native_kit_payload["summary"]["artifact_glob_expansion_count"] >= 1, native_kit_payload["summary"]
if "dist/targets/openai/adapter.json" in native_rows:
assert native_rows["dist/targets/openai/adapter.json"]["source_pattern"] == "dist/targets/*/adapter.json", native_rows
assert native_rows["dist/targets/openai/adapter.json"]["artifact_ref_ready"] is True, native_rows
assert len(native_rows["dist/targets/openai/adapter.json"]["sha256"]) == 64, native_rows
else:
glob_rows = [
item for item in native_kit_payload["artifact_checklist"]
if item["source_pattern"] == "dist/targets/*/adapter.json"
]
assert len(glob_rows) == 1, glob_rows
assert glob_rows[0]["status"] == "glob-no-match", glob_rows[0]
assert glob_rows[0]["artifact_ref_ready"] is False, glob_rows[0]
assert glob_rows[0]["concrete_reference_required"] is True, glob_rows[0]
native_readme = (native_kit_dir / "README.md").read_text(encoding="utf-8")
assert "Glob patterns are expanded into concrete files" in native_readme, native_readme
draft_intake = run_intake("--submissions-dir", str(kit_dir))
assert draft_intake["ok"] is False, draft_intake
assert draft_intake["summary"]["submission_count"] == 1, draft_intake["summary"]
assert draft_intake["summary"]["invalid_submission_count"] == 1, draft_intake["summary"]
assert draft_intake["submissions"][0]["evidence_key"] == "provider-holdout", draft_intake["submissions"]
assert all(item["evidence_key"] != "unknown" for item in draft_intake["submissions"]), draft_intake["submissions"]
spaced_kit_dir = TMP / "submission kit spaced"
spaced_kit_proc = run_kit("--output-dir", str(spaced_kit_dir), "--evidence-key", "provider-holdout")
spaced_kit_payload = json.loads(spaced_kit_proc.stdout)
quoted_spaced_dir = f"'tests/{TMP.name}/submission kit spaced'"
assert quoted_spaced_dir in spaced_kit_payload["commands"]["validate_intake"], spaced_kit_payload["commands"]
assert quoted_spaced_dir in spaced_kit_payload["commands"]["refresh_ledger"], spaced_kit_payload["commands"]
spaced_readme = (spaced_kit_dir / "README.md").read_text(encoding="utf-8")
assert quoted_spaced_dir in spaced_readme, spaced_readme
existing_proc = run_kit("--output-dir", str(kit_dir), "--evidence-key", "provider-holdout")
existing_payload = json.loads(existing_proc.stdout)
assert existing_payload["summary"]["existing_count"] == 1, existing_payload["summary"]
assert existing_payload["files"][0]["status"] == "exists", existing_payload["files"]
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()