diff --git a/reports/adaptation_proposals.json b/reports/adaptation_proposals.json index ce2b81f..ff0accc 100644 --- a/reports/adaptation_proposals.json +++ b/reports/adaptation_proposals.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T07:56:36Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "source_patterns": "reports/user_patterns.json", "pattern_count": 5, diff --git a/reports/adaptation_proposals.md b/reports/adaptation_proposals.md index b0a16ed..2f02219 100644 --- a/reports/adaptation_proposals.md +++ b/reports/adaptation_proposals.md @@ -1,6 +1,6 @@ # Adaptation Proposals -- Generated at: `2026-06-17T07:56:36Z` +- Generated at: `2026-06-17T08:08:10Z` - Pattern report: `reports/user_patterns.json` - Proposal only: `true` - Writes repository files: `false` diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index b3987ae..d96524c 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-17T07:56:35Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 77e410f..330ec34 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,13 +3,13 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "git_status": { "available": true, "dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "generated_dirty": true, - "generated_changed_file_count": 18, + "generated_changed_file_count": 30, "source_dirty": false, "source_changed_file_count": 0, "sample": [ @@ -57,7 +57,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "evidence_bundle_sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935", "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -79,11 +79,11 @@ "public_claim_ready": false, "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 18 + "generated_changed_file_count": 30 }, "public_claim": { "ready": false, @@ -97,10 +97,10 @@ }, "release_lock": { "ready": true, - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "status_scope": "generation-time status before this report is written", "source_changed_file_count": 0, - "generated_changed_file_count": 18, + "generated_changed_file_count": 30, "reason": "only generated evidence artifacts were dirty at generation time" }, "evidence_bundle": { @@ -109,7 +109,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350" + "sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -266,50 +266,50 @@ "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 26940, - "sha256": "a345880034709fcad8741cf20114dba739e2652a76e65d58265f8a87d10c8698" + "bytes": 26277, + "sha256": "6038b6cd6e0667f25cfd69a5e40afa314d9efc3a93753a2a3a0509ae9b516a15" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 21720, - "sha256": "ebbfa25a5211125583f333fdd41a3dea936a54350cc6361dee751fa1da46881b" + "bytes": 20922, + "sha256": "aa262da0ecee7232974ce061a51c8966c310de0b0be9f91eb82bb28bf1c7c0a8" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 94331, - "sha256": "eecbe8b0eed0581af20140cfdccbf4346b9b0ca8022f4a033940a166c2415195" + "bytes": 94341, + "sha256": "95cb56581224aab06be84d0b338d6ba9aa4aef0b1ebd480801f1e2226b873b0d" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 17725, - "sha256": "56f927077bbdb0f601dc56f0d8645432db8e9a8ad78e323828a91dd2f52a1f1b" + "bytes": 17295, + "sha256": "12fbb84926294c4bd05af526fcbe0983db26e449d2c305006094baf27332a0ba" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 87920, - "sha256": "94cce92c415b0348348c9b7dbe9eaf233a1b64ff37b5b98e128e11f2f40412ea" + "bytes": 87257, + "sha256": "f321dc118ee931a1e1510559e5e64774168a58b0446ae62d232009eb42e3b384" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, "bytes": 25549, - "sha256": "94a63a39d2d0989db7a0c6d15f49a5027148654ab30fd197205e7b78173be262" + "sha256": "858ee11bbacb143f648e9c00735bc248a0b9f3c9303dc9e0ad5c0a65d9d6e07f" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, "bytes": 34702, - "sha256": "4dc91fdfc2e54a83aedf5ea99e80481949326d74c6933abd9d85c5f4a8f10c5b" + "sha256": "31b24658c59ee1dee25aec378e72033b042af88dfdf40c325d361d0c9bec2ca7" }, { "label": "world_class_claim_guard", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index e17f33a..4efa5a0 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,11 +1,11 @@ # Benchmark Reproducibility Generated at: `2026-06-17` -Commit: `e39be9d5d40513f362d657b81846b48ce140673b` +Commit: `d845e9513c67b9572fb6c2a8754d3a0051e735ad` Working tree dirty at generation: `true` Source tree dirty at generation: `false` Generated evidence dirty at generation: `true` -Evidence bundle SHA256: `53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350` +Evidence bundle SHA256: `1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935` ## Summary @@ -25,9 +25,9 @@ Evidence bundle SHA256: `53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3ee - world-class source checks: `11` pass / `19` total; `8` blocked - public claim ready: `false` - public claim blockers: `3` -- changed files at generation: `18` +- changed files at generation: `30` - source changed files at generation: `0` -- generated changed files at generation: `18` +- generated changed files at generation: `30` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -53,7 +53,7 @@ This report proves local benchmark reproducibility only. It keeps external provi - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `25` / `25` -- sha256: `53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350` +- sha256: `1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935` ## Methodology Sections @@ -87,13 +87,13 @@ This report proves local benchmark reproducibility only. It keeps external provi | install_simulation | `reports/install_simulation.json` | present | `c1f1122264b8` | | skill_os2_audit | `reports/skill_os2_audit.json` | present | `6362e79a8cfd` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `8eb5e1fbaa75` | -| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `a34588003470` | -| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `ebbfa25a5211` | -| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `eecbe8b0eed0` | -| world_class_submission_review | `reports/world_class_submission_review.json` | present | `56f927077bbd` | -| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `94cce92c415b` | -| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `94a63a39d2d0` | -| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `4dc91fdfc2e5` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `6038b6cd6e06` | +| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `aa262da0ecee` | +| world_class_evidence_preflight | `reports/world_class_evidence_preflight.json` | present | `95cb56581224` | +| world_class_submission_review | `reports/world_class_submission_review.json` | present | `12fbb8492629` | +| world_class_operator_runbook | `reports/world_class_operator_runbook.json` | present | `f321dc118ee9` | +| world_class_operator_runbook_markdown | `reports/world_class_operator_runbook.md` | present | `858ee11bbacb` | +| world_class_operator_runbook_html | `reports/world_class_operator_runbook.html` | present | `31b24658c59e` | | world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `abe7f7d60c00` | ## Reproduction Commands diff --git a/reports/context_budget.json b/reports/context_budget.json index dcb5d48..52a2c25 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,9 +6,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 805, - "other_text_tokens": 1152749, + "other_text_tokens": 1153001, "estimated_initial_load_tokens": 998, - "estimated_total_text_tokens": 1153554, + "estimated_total_text_tokens": 1153806, "deferred_resource_tokens": 523653, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index ea8461f..bf04bcf 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -357,7 +357,7 @@ "actual": { "available": true, "clean": false, - "changed_file_count": 15 + "changed_file_count": 28 }, "paths": [ "reports/benchmark_reproducibility.json" @@ -398,8 +398,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "e39be9d5d40513f362d657b81846b48ce140673b", - "actual": "e39be9d5d40513f362d657b81846b48ce140673b", + "expected": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", + "actual": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -544,8 +544,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "e39be9d5d40513f362d657b81846b48ce140673b", - "actual": "e39be9d5d40513f362d657b81846b48ce140673b", + "expected": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", + "actual": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" diff --git a/reports/review-studio.html b/reports/review-studio.html index 4baadfc..355b26d 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -772,12 +772,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 10; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

3 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

153 scripts scanned; secrets found

Py Compat0

232 files scanned for Python 3.11

Arch Debt0

711 largest lines; 0 watchlist; 5 early; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 1 invalid

Claim Guard0

182 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

689 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 10; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

3 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

153 scripts scanned; secrets found

Py Compat0

232 files scanned for Python 3.11

Arch Debt0

711 largest lines; 0 watchlist; 5 early; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock false

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

182 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

689 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 998/1000; deferred 523653/120000; top deferred scripts 462350; resource governance governed; quality density 140.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 153 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 232 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

229 Python files; 0 hotspots; 0 watchlist files; 5 early watch files; 0 blockers; largest 711 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 10; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 998/1000; deferred 523653/120000; top deferred scripts 462350; resource governance governed; quality density 140.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 153 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 232 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

229 Python files; 0 hotspots; 0 watchlist files; 5 early watch files; 0 blockers; largest 711 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 11/19 pass; 8 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -874,14 +874,14 @@
-

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

+

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
-

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
是
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
-

周度队列

决策
curator-review
Week Id
2026-W25
日报数
2
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
是
Writes Source Files
否
Auto Patch Enabled
否
+

日常运维

决策
proposal-review
提案数
5
Approval Count
0
Pending Review Count
0
发布锁
否
世界级
否
Writes Source Files
否
Auto Patch Enabled
否
+

周度队列

决策
curator-review
Week Id
2026-W25
日报数
2
唯一机会
5
待批准复核
1
提案复核
4
Top Score
88
发布锁
否
Writes Source Files
否
Auto Patch Enabled
否
@@ -899,13 +899,13 @@

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 10; token-observed 10
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 10; timing_observed_count: 10; token_observed_count: 10; accepted: True
提交态
status: invalid-contract; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: True; privacy_contract_satisfied: True

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 10 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 10 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; reviewer_metadata_present: False; reason_required: True; blind_review_attested: False; raw_content_excluded_attested: True; reviewer_reason_required_attested: True; ready_for_human_evidence: False; raw_content_allowed: False; raw_content_path_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 10; token-observed 10
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 10; timing_observed_count: 10; token_observed_count: 10; accepted: True
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 10 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 10 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; reviewer_metadata_present: False; reason_required: True; blind_review_attested: False; raw_content_excluded_attested: True; reviewer_reason_required_attested: True; ready_for_human_evidence: False; raw_content_allowed: False; raw_content_path_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice and reason for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
  • Reviewer metadatareviewer_metadata_present: False / trueRecord reviewer and reviewed_at before adjudication can count.
  • Reason requiredreason_required: True / trueKeep reason mandatory for every imported or direct reviewer decision.
  • Blind review attestedblind_review_attested: False / trueSet reviewer_attestation only after choices are completed before opening the answer key.
  • Raw content attestedraw_content_excluded_attested: True / trueAttest that reviewer decisions exclude raw prompts, outputs, transcripts, messages, and private user content.
  • Raw content blockedraw_content_allowed: False / falseAdjudication evidence should store prompt hashes and reviewer metadata, not raw prompts, outputs, transcripts, or messages.
  • Human evidence readyready_for_human_evidence: False / trueComplete all reviewer decisions with metadata and rationale, plus blind-review attestation and integrity fingerprints.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.

提交清单

每张卡片给出模板、提交文件、准备命令、校验命令、收集要求、通过条件和隐私边界；只有真实 provider、真人、原生权限或真实客户端证据通过 intake 后，才进入 ledger review。

-
fix-submission · external

Provider Holdout

Submission exists but failed intake validation.

负责人
operator with provider credentials
模板
evidence/world_class/templates/provider-holdout.intake.json
提交
evidence/world_class/submissions/provider-holdout.json
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • provider-backed model run
  • observed timing
  • observed token metadata

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

隐私边界

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
awaiting-submission · human

Human Adjudication

No real evidence submission has been provided yet.

负责人
human reviewer
模板
evidence/world_class/templates/human-adjudication.intake.json
提交
evidence/world_class/submissions/human-adjudication.json
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

隐私边界

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.
awaiting-submission · external

Native Permission Enforcement

No real evidence submission has been provided yet.

负责人
target client or installer integrator
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
提交
evidence/world_class/submissions/native-permission-enforcement.json
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

隐私边界

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
awaiting-submission · external

Native Client Telemetry

No real evidence submission has been provided yet.

负责人
Browser/Chrome/IDE/provider client integrator
模板
evidence/world_class/templates/native-client-telemetry.intake.json
提交
evidence/world_class/submissions/native-client-telemetry.json
下一步
Install a real client against the native host and import production metadata-only events.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real external client source
  • metadata-only event
  • local-first import path

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

隐私边界

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
awaiting-submission · external

Provider Holdout

No real evidence submission has been provided yet.

负责人
operator with provider credentials
模板
evidence/world_class/templates/provider-holdout.intake.json
提交
evidence/world_class/submissions/provider-holdout.json
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • provider-backed model run
  • observed timing
  • observed token metadata

执行步骤

  • Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value.
  • For OpenAI Responses: python3 scripts/yao.py output-exec --provider-runner openai --provider-model ${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini} --timeout-seconds 60
  • For DeepSeek Chat Completions: python3 scripts/yao.py output-exec --provider-runner deepseek --provider-model deepseek-v4-flash --provider-api-format chat-completions --provider-thinking disabled --api-key-env DEEPSEEK_API_KEY --timeout-seconds 120
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

隐私边界

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
awaiting-submission · human

Human Adjudication

No real evidence submission has been provided yet.

负责人
human reviewer
模板
evidence/world_class/templates/human-adjudication.intake.json
提交
evidence/world_class/submissions/human-adjudication.json
下一步
Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with reviewer, reviewed_at, case_id, winner_variant, confidence, required reason, and truthful reviewer_attestation only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --blind-review-attested --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/output_review_adjudication.json summary.reviewer_metadata_present is true
  • reports/output_review_adjudication.json summary.blind_review_attested is true
  • reports/output_review_adjudication.json review_integrity.blind_pack_sha256 exists and matches reports/output_review_decisions.json
  • reports/output_review_adjudication.json pairs and reviewer_checklist store prompt_sha256, not raw prompt text
  • reports/output_review_adjudication.json summary.ready_for_human_evidence is true
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

隐私边界

  • Reviewer decisions should not include raw user data or private customer detail.
  • Reviewer reasons must be rubric-based and must not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • The adjudication evidence stores prompt_sha256 instead of raw prompt text.
  • The decision and adjudication artifacts preserve blind_pack_sha256 so reviewers can audit exactly which pack was judged.
  • Keep the answer key separate until after decisions are recorded.
awaiting-submission · external

Native Permission Enforcement

No real evidence submission has been provided yet.

负责人
target client or installer integrator
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
提交
evidence/world_class/submissions/native-permission-enforcement.json
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

隐私边界

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
awaiting-submission · external

Native Client Telemetry

No real evidence submission has been provided yet.

负责人
Browser/Chrome/IDE/provider client integrator
模板
evidence/world_class/templates/native-client-telemetry.intake.json
提交
evidence/world_class/submissions/native-client-telemetry.json
下一步
Install a real client against the native host and import production metadata-only events.

操作命令

  • prepare_submissionpython3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • validate_intakepython3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • submission_reviewpython3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • refresh_ledgerpython3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
  • guard_claimpython3 scripts/yao.py world-class-claim-guard .

收集要求

  • real external client source
  • metadata-only event
  • local-first import path

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

通过条件

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

隐私边界

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -924,7 +924,7 @@
-

证据入口

Schema
是
模板数
4
模板通过
4
提交数
1
有效提交
0
无效提交
1
可收证据
否
可审台账
否
Ready To Claim World Class
否
+

证据入口

Schema
是
模板数
4
模板通过
4
提交数
0
有效提交
0
无效提交
0
可收证据
是
可审台账
否
Ready To Claim World Class
否

入口边界

intake 只校验证据包格式、来源、隐私和反过度声明；只有 ledger 看到真实 provider、真人、原生权限或真实客户端结果后，world-class 才能进入完成审计。

diff --git a/reports/review-studio.json b/reports/review-studio.json index d45f08a..d470844 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -185,7 +185,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", + "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock false; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock false", "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", "link": "adoption_drift_report.md", "review_action_id": "", @@ -652,7 +652,7 @@ "label": "Provider Holdout", "category": "external", "status": "pending", - "readiness": "fix-submission", + "readiness": "awaiting-submission", "current": "model-executed 10; token-observed 10", "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", @@ -2480,7 +2480,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "evidence_bundle_sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935", "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -2502,13 +2502,13 @@ "public_claim_ready": false, "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 18 + "generated_changed_file_count": 30 }, - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -3141,8 +3141,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -3237,26 +3237,14 @@ "source_pass_count": 3, "source_blocked_count": 0, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -6343,13 +6331,13 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "git_status": { "available": true, "dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "generated_dirty": true, - "generated_changed_file_count": 18, + "generated_changed_file_count": 30, "source_dirty": false, "source_changed_file_count": 0, "sample": [ @@ -6397,7 +6385,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "evidence_bundle_sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935", "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -6419,11 +6407,11 @@ "public_claim_ready": false, "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 18 + "generated_changed_file_count": 30 }, "public_claim": { "ready": false, @@ -6437,10 +6425,10 @@ }, "release_lock": { "ready": true, - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "status_scope": "generation-time status before this report is written", "source_changed_file_count": 0, - "generated_changed_file_count": 18, + "generated_changed_file_count": 30, "reason": "only generated evidence artifacts were dirty at generation time" }, "evidence_bundle": { @@ -6449,7 +6437,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350" + "sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -6606,50 +6594,50 @@ "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 26940, - "sha256": "a345880034709fcad8741cf20114dba739e2652a76e65d58265f8a87d10c8698" + "bytes": 26277, + "sha256": "6038b6cd6e0667f25cfd69a5e40afa314d9efc3a93753a2a3a0509ae9b516a15" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, - "bytes": 21720, - "sha256": "ebbfa25a5211125583f333fdd41a3dea936a54350cc6361dee751fa1da46881b" + "bytes": 20922, + "sha256": "aa262da0ecee7232974ce061a51c8966c310de0b0be9f91eb82bb28bf1c7c0a8" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 94331, - "sha256": "eecbe8b0eed0581af20140cfdccbf4346b9b0ca8022f4a033940a166c2415195" + "bytes": 94341, + "sha256": "95cb56581224aab06be84d0b338d6ba9aa4aef0b1ebd480801f1e2226b873b0d" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 17725, - "sha256": "56f927077bbdb0f601dc56f0d8645432db8e9a8ad78e323828a91dd2f52a1f1b" + "bytes": 17295, + "sha256": "12fbb84926294c4bd05af526fcbe0983db26e449d2c305006094baf27332a0ba" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 87920, - "sha256": "94cce92c415b0348348c9b7dbe9eaf233a1b64ff37b5b98e128e11f2f40412ea" + "bytes": 87257, + "sha256": "f321dc118ee931a1e1510559e5e64774168a58b0446ae62d232009eb42e3b384" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, "bytes": 25549, - "sha256": "94a63a39d2d0989db7a0c6d15f49a5027148654ab30fd197205e7b78173be262" + "sha256": "858ee11bbacb143f648e9c00735bc248a0b9f3c9303dc9e0ad5c0a65d9d6e07f" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, "bytes": 34702, - "sha256": "4dc91fdfc2e54a83aedf5ea99e80481949326d74c6933abd9d85c5f4a8f10c5b" + "sha256": "31b24658c59ee1dee25aec378e72033b042af88dfdf40c325d361d0c9bec2ca7" }, { "label": "world_class_claim_guard", @@ -18127,9 +18115,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 805, - "other_text_tokens": 1152749, + "other_text_tokens": 1153001, "estimated_initial_load_tokens": 998, - "estimated_total_text_tokens": 1153554, + "estimated_total_text_tokens": 1153806, "deferred_resource_tokens": 523653, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ @@ -20107,7 +20095,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-17T07:56:35Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -20191,7 +20179,7 @@ "daily_skillops": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T07:56:36Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "decision": "proposal-review", "source_supplied": false, @@ -20204,7 +20192,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -20221,7 +20209,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -20670,7 +20658,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true }, "actions": [ @@ -20705,7 +20693,7 @@ "weekly_curator": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T07:56:36Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "decision": "curator-review", "week_id": "2026-W25", @@ -20719,7 +20707,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -20739,7 +20727,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -20992,7 +20980,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 @@ -21170,7 +21158,7 @@ "adaptation_proposals": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T07:56:36Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "source_patterns": "reports/user_patterns.json", "pattern_count": 5, @@ -21580,8 +21568,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -21597,8 +21585,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -21719,26 +21707,14 @@ "source_pass_count": 3, "source_blocked_count": 0, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { @@ -22181,7 +22157,7 @@ }, "world_class_evidence_intake": { "schema_version": "1.0", - "ok": false, + "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", "summary": { @@ -22189,17 +22165,17 @@ "ledger_entry_count": 4, "template_count": 4, "template_pass_count": 4, - "submission_count": 1, + "submission_count": 0, "valid_submission_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "operator_checklist_count": 4, "operator_checklist_ready_count": 0, "valid_packet_source_incomplete_count": 0, - "ready_for_external_collection": false, + "ready_for_external_collection": true, "ready_for_ledger_review": false, "ready_to_claim_world_class": false, "overclaim_guard_active": true, - "decision": "fix-intake" + "decision": "awaiting-submissions" }, "schema": "evidence/world_class/intake.schema.json", "templates": [ @@ -22260,38 +22236,17 @@ "errors": [] } ], - "submissions": [ - { - "path": "evidence/world_class/submissions/provider-holdout.json", - "evidence_key": "provider-holdout", - "status": "fail", - "template_only": false, - "artifact_integrity": { - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "required_artifact_count": 1, - "required_artifact_verified_count": 1 - }, - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ] - } - ], + "submissions": [], "operator_checklist": [ { "evidence_key": "provider-holdout", "label": "Provider Holdout", "category": "external", "owner": "operator with provider credentials", - "readiness": "fix-submission", - "blocking_reason": "Submission exists but failed intake validation.", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", "template_status": "pass", - "submission_status": "fail", + "submission_status": "missing", "source_accepted": true, "observed_state": { "model_executed_count": 10, @@ -22660,8 +22615,8 @@ "owner": "operator with provider credentials", "status": "blocked", "ledger_status": "pending", - "intake_readiness": "fix-submission", - "review_state": "fix-submission", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", "source_accepted": true, "prechecks": [ { @@ -24638,19 +24593,19 @@ }, "world_class_submission_review": { "schema_version": "1.0", - "ok": false, + "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "decision": "fix-submissions", + "decision": "awaiting-submissions", "ready_to_claim_world_class": false, "review_item_count": 4, "accepted_count": 0, - "awaiting_submission_count": 3, + "awaiting_submission_count": 4, "valid_packet_source_incomplete_count": 0, "ready_for_ledger_review_count": 0, - "fix_submission_count": 1, + "fix_submission_count": 0, "unmatched_submission_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -24658,18 +24613,18 @@ "summary": { "review_item_count": 4, "accepted_count": 0, - "awaiting_submission_count": 3, + "awaiting_submission_count": 4, "valid_packet_source_incomplete_count": 0, "ready_for_ledger_review_count": 0, - "fix_submission_count": 1, + "fix_submission_count": 0, "unmatched_submission_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, - "decision": "fix-submissions" + "decision": "awaiting-submissions" }, "report_contract": { "schema_version": "1.0", @@ -24704,22 +24659,16 @@ "category": "external", "owner": "operator with provider credentials", "ledger_status": "pending", - "submission_status": "invalid-contract", - "intake_status": "fail", + "submission_status": "missing", + "intake_status": "missing", "source_accepted": true, - "review_state": "fix-submission", - "blocking_reason": "Submission exists but fails the ledger submission contract.", + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "intake_errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], "observed_state": { "model_executed_count": 10, "timing_observed_count": 10, @@ -25093,10 +25042,10 @@ "evidence_item_count": 4, "pending_count": 4, "accepted_count": 0, - "awaiting_submission_count": 3, + "awaiting_submission_count": 4, "ready_for_ledger_review_count": 0, "valid_packet_source_incomplete_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -25137,8 +25086,8 @@ "category": "external", "owner": "operator with provider credentials", "ledger_status": "pending", - "intake_readiness": "fix-submission", - "review_state": "fix-submission", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", "source_accepted": true, "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "current": "model-executed 10; token-observed 10", @@ -25150,7 +25099,7 @@ "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" ], - "blocking_reason": "Submission exists but fails the ledger submission contract.", + "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", "commands": { @@ -25285,26 +25234,14 @@ "phase_queue_blocked_count": 1, "phase_queue_counts_as_completion": false, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 3348f7d..bb03b04 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -1017,7 +1017,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "evidence_bundle_sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935", "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -1039,13 +1039,13 @@ "public_claim_ready": false, "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 18 + "generated_changed_file_count": 30 }, - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -1678,8 +1678,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -1774,26 +1774,14 @@ "source_pass_count": 3, "source_blocked_count": 0, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index 441e89c..e5bf5f5 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -1021,7 +1021,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "evidence_bundle_sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935", "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -1043,13 +1043,13 @@ "public_claim_ready": false, "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 18 + "generated_changed_file_count": 30 }, - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -1682,8 +1682,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -1778,26 +1778,14 @@ "source_pass_count": 3, "source_blocked_count": 0, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { diff --git a/reports/skill-overview.json b/reports/skill-overview.json index f386563..80fe173 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -1016,7 +1016,7 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "53b15993b4b79fd9bf3d5016ca565969d27a8a2604fe659a876f3eebe7292350", + "evidence_bundle_sha256": "1a1f43932b87e208a18729f742be830c4c6554be2bde13d65591c525cbe51935", "source_contract_sha256": "7fc324e26d614c46b350c2b3e20cb87a86f967d3a74d345bc0baf73d78d8b111", "archive_sha256": "7db44f059844b71930ea8194a72ba1a316116ca970620de5550d60616e8f23c2", "output_case_count": 5, @@ -1038,13 +1038,13 @@ "public_claim_ready": false, "public_claim_blocker_count": 3, "working_tree_dirty": true, - "changed_file_count": 18, + "changed_file_count": 30, "source_tree_dirty": false, "source_changed_file_count": 0, "generated_tree_dirty": true, - "generated_changed_file_count": 18 + "generated_changed_file_count": 30 }, - "commit": "e39be9d5d40513f362d657b81846b48ce140673b", + "commit": "d845e9513c67b9572fb6c2a8754d3a0051e735ad", "missing_artifacts": [], "limitations": [ "The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.", @@ -1677,8 +1677,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -1773,26 +1773,14 @@ "source_pass_count": 3, "source_blocked_count": 0, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { diff --git a/reports/skillops/daily/2026-06-17.json b/reports/skillops/daily/2026-06-17.json index 30e6c32..7c889a9 100644 --- a/reports/skillops/daily/2026-06-17.json +++ b/reports/skillops/daily/2026-06-17.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T07:56:36Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "decision": "proposal-review", "source_supplied": false, @@ -14,7 +14,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -31,7 +31,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "writes_source_files": false, "auto_patch_enabled": false, @@ -480,7 +480,7 @@ "local_blueprint_ready": true, "public_world_class_ready": false, "world_class_pending_count": 4, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true }, "actions": [ diff --git a/reports/skillops/daily/2026-06-17.md b/reports/skillops/daily/2026-06-17.md index 38ae607..5766ec2 100644 --- a/reports/skillops/daily/2026-06-17.md +++ b/reports/skillops/daily/2026-06-17.md @@ -1,6 +1,6 @@ # Daily SkillOps Report -Generated at: `2026-06-17T07:56:36Z` +Generated at: `2026-06-17T08:08:10Z` ## Summary @@ -14,7 +14,7 @@ Generated at: `2026-06-17T07:56:36Z` - local blueprint ready: `true` - public world-class ready: `false` - world-class pending: `4` -- release lock ready: `true` +- release lock ready: `false` - evidence consistency ok: `true` This report is an operations cockpit for explicit-source SkillOps. It does not scan private logs, write source files, apply patches, or count as world-class external or human evidence. diff --git a/reports/skillops/weekly/2026-W25.json b/reports/skillops/weekly/2026-W25.json index 431519b..6a98f95 100644 --- a/reports/skillops/weekly/2026-W25.json +++ b/reports/skillops/weekly/2026-W25.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17T07:56:36Z", + "generated_at": "2026-06-17T08:08:10Z", "skill_dir": ".", "decision": "curator-review", "week_id": "2026-W25", @@ -15,7 +15,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -35,7 +35,7 @@ "top_score": 88, "skill_count": 12, "actionable_portfolio_issue_count": 0, - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4, @@ -288,7 +288,7 @@ "no_route_opportunity_count": 0 }, "release_state": { - "release_lock_ready": true, + "release_lock_ready": false, "evidence_consistency_ok": true, "public_world_class_ready": false, "world_class_pending_count": 4 diff --git a/reports/skillops/weekly/2026-W25.md b/reports/skillops/weekly/2026-W25.md index b071f13..b78002f 100644 --- a/reports/skillops/weekly/2026-W25.md +++ b/reports/skillops/weekly/2026-W25.md @@ -1,6 +1,6 @@ # Weekly SkillOps Curator Report -Generated at: `2026-06-17T07:56:36Z` +Generated at: `2026-06-17T08:08:10Z` Week: `2026-W25` ## Summary @@ -13,7 +13,7 @@ Week: `2026-W25` - top_score: `88` - skill_count: `12` - actionable_portfolio_issue_count: `0` -- release_lock_ready: `True` +- release_lock_ready: `False` - evidence_consistency_ok: `True` - public_world_class_ready: `False` - world_class_pending_count: `4` diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index fea69f1..c90e7d1 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -1,6 +1,6 @@ { "schema_version": "1.0", - "ok": false, + "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", "summary": { @@ -8,17 +8,17 @@ "ledger_entry_count": 4, "template_count": 4, "template_pass_count": 4, - "submission_count": 1, + "submission_count": 0, "valid_submission_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "operator_checklist_count": 4, "operator_checklist_ready_count": 0, "valid_packet_source_incomplete_count": 0, - "ready_for_external_collection": false, + "ready_for_external_collection": true, "ready_for_ledger_review": false, "ready_to_claim_world_class": false, "overclaim_guard_active": true, - "decision": "fix-intake" + "decision": "awaiting-submissions" }, "schema": "evidence/world_class/intake.schema.json", "templates": [ @@ -79,38 +79,17 @@ "errors": [] } ], - "submissions": [ - { - "path": "evidence/world_class/submissions/provider-holdout.json", - "evidence_key": "provider-holdout", - "status": "fail", - "template_only": false, - "artifact_integrity": { - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "required_artifact_count": 1, - "required_artifact_verified_count": 1 - }, - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ] - } - ], + "submissions": [], "operator_checklist": [ { "evidence_key": "provider-holdout", "label": "Provider Holdout", "category": "external", "owner": "operator with provider credentials", - "readiness": "fix-submission", - "blocking_reason": "Submission exists but failed intake validation.", + "readiness": "awaiting-submission", + "blocking_reason": "No real evidence submission has been provided yet.", "template_status": "pass", - "submission_status": "fail", + "submission_status": "missing", "source_accepted": true, "observed_state": { "model_executed_count": 10, diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index f46de6a..50426d6 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -4,14 +4,14 @@ Generated at: `2026-06-17` ## Summary -- decision: `fix-intake` +- decision: `awaiting-submissions` - schema present: `true` - templates: `4` / `4` -- submissions: `0` valid / `1` total -- invalid submissions: `1` +- submissions: `0` valid / `0` total +- invalid submissions: `0` - valid packet but source incomplete: `0` - operator checklist: `0` ready / `4` total -- ready for external collection: `false` +- ready for external collection: `true` - ready for ledger review: `false` - ready to claim world-class: `false` - overclaim guard active: `true` @@ -31,21 +31,21 @@ This report validates the intake contract for human and external evidence. A val | Evidence | Status | Path | Artifacts | Errors | | --- | --- | --- | --- | --- | -| `provider-holdout` | `fail` | `evidence/world_class/submissions/provider-holdout.json` | 1 existing / 1 sha256 verified / 1 required verified / 1 refs | attestation.ledger_reviewer_approved must be true for a real submission; attestation.ledger_reviewer is required; attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ; attestation.ledger_reviewer must be different from submitted_by; attestation.ledger_reviewed_at must be at or after submitted_at | +| `none` | `n/a` | none | none | none | ## Operator Checklist | Evidence | Readiness | Submission | Next action | | --- | --- | --- | --- | -| `provider-holdout` | `fix-submission` | `fail` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `provider-holdout` | `awaiting-submission` | `missing` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | `human-adjudication` | `awaiting-submission` | `missing` | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | | `native-permission-enforcement` | `awaiting-submission` | `missing` | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | `native-client-telemetry` | `awaiting-submission` | `missing` | Install a real client against the native host and import production metadata-only events. | ### Provider Holdout -- readiness: `fix-submission` -- blocking reason: Submission exists but failed intake validation. +- readiness: `awaiting-submission` +- blocking reason: No real evidence submission has been provided yet. - owner: operator with provider credentials - template: `evidence/world_class/templates/provider-holdout.intake.json` - submission: `evidence/world_class/submissions/provider-holdout.json` diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index ef298e3..89d6c6e 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -13,8 +13,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -30,8 +30,8 @@ "external_pending_count": 3, "submitted_entry_count": 0, "reviewer_approved_submission_count": 0, - "missing_submission_count": 3, - "invalid_submission_count": 1, + "missing_submission_count": 4, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -152,26 +152,14 @@ "source_pass_count": 3, "source_blocked_count": 0, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index d1e389f..14ee336 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -18,7 +18,7 @@ Generated at: `2026-06-17` - reviewer approved submissions: `0` - submitted but pending: `0` - source accepted without valid submission: `1` -- invalid submissions: `1` +- invalid submissions: `0` - overclaim guard active: `true` This ledger records the current evidence state. It requires both passing source evidence and a validated intake submission with artifact SHA-256 checks before accepting an item. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence. @@ -27,7 +27,7 @@ This ledger records the current evidence state. It requires both passing source | Evidence | Status | Submission | Category | Current | Next action | | --- | --- | --- | --- | --- | --- | -| `provider-holdout` | `pending` | `invalid-contract` | `external` | model-executed 10; token-observed 10 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `provider-holdout` | `pending` | `missing` | `external` | model-executed 10; token-observed 10 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | `human-adjudication` | `pending` | `missing` | `human` | 0/5 decisions; pending 5 | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | | `native-permission-enforcement` | `pending` | `missing` | `external` | native-enforced targets 0; installer-enforced targets 4 | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | `native-client-telemetry` | `pending` | `missing` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. | @@ -38,7 +38,7 @@ This ledger records the current evidence state. It requires both passing source - source status: `pass` - observed state: `{"model_executed_count": 10, "timing_observed_count": 10, "token_observed_count": 10, "accepted": true}` - source checks: `3` pass / `3` total -- submission state: `{"status": "invalid-contract", "path": "evidence/world_class/submissions/provider-holdout.json", "submitted_by": "Codex operator using DeepSeek official API", "submitted_at": "2026-06-17", "artifact_ref_count": 1, "artifact_existing_count": 1, "artifact_sha256_verified_count": 1, "attested_real_evidence": true, "reviewer_or_operator_identity_present": true, "privacy_contract_satisfied": true, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "errors": ["attestation.ledger_reviewer_approved must be true for a real submission", "attestation.ledger_reviewer is required", "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", "attestation.ledger_reviewer must be different from submitted_by", "attestation.ledger_reviewed_at must be at or after submitted_at"], "ledger_counts_as_completion": false}` +- submission state: `{"status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", "artifact_ref_count": 0, "attested_real_evidence": false, "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", "ledger_counts_as_completion": false}` ### Provenance Requirements diff --git a/reports/world_class_evidence_preflight.html b/reports/world_class_evidence_preflight.html index cb09042..8c340ac 100644 --- a/reports/world_class_evidence_preflight.html +++ b/reports/world_class_evidence_preflight.html @@ -147,8 +147,8 @@
Evidence
provider-holdout
Ledger
pending
-
Intake
fix-submission
-
Review
fix-submission
+
Intake
awaiting-submission
+
Review
awaiting-submission
Draft
evidence/world_class/submissions/provider-holdout.json
diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index ac62f37..9b3b237 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -60,8 +60,8 @@ "owner": "operator with provider credentials", "status": "blocked", "ledger_status": "pending", - "intake_readiness": "fix-submission", - "review_state": "fix-submission", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", "source_accepted": true, "prechecks": [ { diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index fb7579e..048f954 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -57,7 +57,7 @@ Phase queue rows group the same repair checklist into operator execution phases. | Evidence | Status | Intake | Review | Next action | | --- | --- | --- | --- | --- | -| `provider-holdout` | `blocked` | `fix-submission` | `fix-submission` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | +| `provider-holdout` | `blocked` | `awaiting-submission` | `awaiting-submission` | Set one provider API key in the operator shell, such as OPENAI_API_KEY or DEEPSEEK_API_KEY; never commit or print the value. | | `human-adjudication` | `ready-for-human-review` | `awaiting-submission` | `awaiting-submission` | Assign a real reviewer identity before claiming human adjudication. | | `native-permission-enforcement` | `blocked` | `awaiting-submission` | `awaiting-submission` | Attach a real target-client or external installer runtime guard; metadata fallback is not enough. | | `native-client-telemetry` | `blocked` | `awaiting-submission` | `awaiting-submission` | Install a real Browser, Chrome, IDE, or provider client that emits metadata-only events. | diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index 6731d2a..eab3493 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -68,14 +68,14 @@ Evidence Operations

World-Class Operator Runbook

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

-
Pending4
Awaiting3
Ready0
Source11/19
Queue2/2
Steps6/6
Gate4/5
Blocked8
Invalid1
+
Pending4
Awaiting4
Ready0
Source11/19
Queue2/2
Steps6/6
Gate4/5
Blocked8
Invalid0

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.

Coordination Plan

These steps tell the assistant what to run and where the user must supply real provider, reviewer, client, or telemetry input. Every row is operational guidance only and keeps counts_as_completion false.

StepEvidenceOwnerUserUser actionAssistant actionCommandPass condition
prepare-evidence-sessionallassistant + usertrueConfirm provider access, reviewer availability, target client path, and telemetry client path before collection starts.Run preflight and prepare submission drafts without accepting them as evidence.python3 scripts/yao.py world-class-preflight . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-kit . --output-dir evidence/world_class/submissions --prefill-artifactsPreflight lists the same pending evidence keys and no credential values are printed.
collect-provider-holdoutprovider-holdoutassistant + operator with provider credentialstrueProvide the selected provider API key through an environment variable and confirm the provider, model, endpoint, and API format to use.Run provider-backed output execution, verify aggregate timing and token metadata, then prepare the evidence packet.python3 scripts/yao.py output-exec . --provider-runner <openai|deepseek> --provider-model <model> --timeout-seconds 60reports/output_execution_runs.json has model_executed_count > 0 and token_observed_count > 0.
collect-human-adjudicationhuman-adjudicationhuman reviewer + assistanttrueOpen the blind review kit, choose winners for all pairs, add reviewer metadata and reasons, and keep the answer key hidden until decisions are saved.Generate the review kit, import decisions, validate integrity, and prepare the human evidence packet.python3 scripts/yao.py output-review-kit . && python3 scripts/yao.py output-review .reports/output_review_adjudication.json has pending_count == 0 and ready_for_human_evidence == true.
collect-native-permission-enforcementnative-permission-enforcementtarget client or installer integrator + assistanttrueSelect a real target client or external installer guard that can enforce declared capabilities instead of metadata-only fallback.Run runtime permission probes, package verification, install simulation, and prepare the native enforcement evidence packet.python3 scripts/yao.py runtime-permissions . --package-dir distreports/runtime_permission_probes.json has native_enforcement_count > 0 and failure_count == 0.
collect-native-client-telemetrynative-client-telemetryreal client integrator + assistanttrueInstall the native host manifest in a real Browser, Chrome, IDE, or provider client and trigger a metadata-only event.Generate native host assets, import the external event JSONL, refresh adoption drift, and prepare the telemetry evidence packet.python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl --source externalreports/adoption_drift_report.json has source_types.external > 0 and adoption_sample_count > 0.
review-and-release-gateallassistant + ledger reviewertrueApprove only validated evidence packets and confirm the release wording after the claim guard passes.Run intake, submission review, ledger, claim guard, benchmark, evidence consistency, Review Studio, and CI before final publish.python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard . && make ci-testLedger ready_to_claim_world_class, benchmark public_claim_ready, claim guard violation_count == 0, Review Studio has no blockers, and CI passes.

Phase Queue

-

Evidence Items

-
external · fix-submission

Provider Holdout

-

Submission exists but fails the ledger submission contract.

+

Evidence Items

+
external · awaiting-submission

Provider Holdout

+

No evidence packet has been submitted for review.

Owner
operator with provider credentials
Ledger
pending
diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index b61f025..7a3f222 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -7,10 +7,10 @@ "evidence_item_count": 4, "pending_count": 4, "accepted_count": 0, - "awaiting_submission_count": 3, + "awaiting_submission_count": 4, "ready_for_ledger_review_count": 0, "valid_packet_source_incomplete_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -51,8 +51,8 @@ "category": "external", "owner": "operator with provider credentials", "ledger_status": "pending", - "intake_readiness": "fix-submission", - "review_state": "fix-submission", + "intake_readiness": "awaiting-submission", + "review_state": "awaiting-submission", "source_accepted": true, "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", "current": "model-executed 10; token-observed 10", @@ -64,7 +64,7 @@ "Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.", "python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions" ], - "blocking_reason": "Submission exists but fails the ledger submission contract.", + "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", "template_path": "evidence/world_class/templates/provider-holdout.intake.json", "commands": { @@ -199,26 +199,14 @@ "phase_queue_blocked_count": 1, "phase_queue_counts_as_completion": false, "submission_state": { - "status": "invalid-contract", + "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "artifact_existing_count": 1, - "artifact_sha256_verified_count": 1, - "attested_real_evidence": true, - "reviewer_or_operator_identity_present": true, - "privacy_contract_satisfied": true, + "artifact_ref_count": 0, + "attested_real_evidence": false, + "privacy_contract_satisfied": false, "ledger_reviewer_approved": false, "ledger_reviewer": "", "ledger_reviewed_at": "", - "errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], "ledger_counts_as_completion": false }, "anti_overclaim": { diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index 92df3ed..a69f6e0 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -9,7 +9,7 @@ Generated at: `2026-06-17` - runbook counts as completion: `false` - evidence items: `4` - pending: `4` -- awaiting submission: `3` +- awaiting submission: `4` - ready for ledger review: `0` - phase queue: `2` blocked / `2` phases - phase queue rows: `12` @@ -52,7 +52,7 @@ This runbook coordinates evidence collection only. It does not accept submission | Evidence | Ledger | Intake | Review | Blocked checks | Next source action | Owner | | --- | --- | --- | --- | ---: | --- | --- | -| `provider-holdout` | `pending` | `fix-submission` | `fix-submission` | `0` | none | operator with provider credentials | +| `provider-holdout` | `pending` | `awaiting-submission` | `awaiting-submission` | `0` | none | operator with provider credentials | | `human-adjudication` | `pending` | `awaiting-submission` | `awaiting-submission` | `5` | Record a reviewer choice and reason for every pair. | human reviewer | | `native-permission-enforcement` | `pending` | `awaiting-submission` | `awaiting-submission` | `1` | Collect real target-client or external runtime guard proof. | target client or installer integrator | | `native-client-telemetry` | `pending` | `awaiting-submission` | `awaiting-submission` | `2` | Import at least one metadata-only event from a real client. | Browser/Chrome/IDE/provider client integrator | @@ -60,7 +60,7 @@ This runbook coordinates evidence collection only. It does not accept submission ## Provider Holdout - objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata. -- blocking reason: Submission exists but fails the ledger submission contract. +- blocking reason: No evidence packet has been submitted for review. - blocked source checks: `0` - repair rows: `1` blocked - phase queue: `1` blocked phases diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 0d44ec7..50689d7 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -1,18 +1,18 @@ { "schema_version": "1.0", - "ok": false, + "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "decision": "fix-submissions", + "decision": "awaiting-submissions", "ready_to_claim_world_class": false, "review_item_count": 4, "accepted_count": 0, - "awaiting_submission_count": 3, + "awaiting_submission_count": 4, "valid_packet_source_incomplete_count": 0, "ready_for_ledger_review_count": 0, - "fix_submission_count": 1, + "fix_submission_count": 0, "unmatched_submission_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, @@ -20,18 +20,18 @@ "summary": { "review_item_count": 4, "accepted_count": 0, - "awaiting_submission_count": 3, + "awaiting_submission_count": 4, "valid_packet_source_incomplete_count": 0, "ready_for_ledger_review_count": 0, - "fix_submission_count": 1, + "fix_submission_count": 0, "unmatched_submission_count": 0, - "invalid_submission_count": 1, + "invalid_submission_count": 0, "source_check_count": 19, "source_pass_count": 11, "source_blocked_count": 8, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, - "decision": "fix-submissions" + "decision": "awaiting-submissions" }, "report_contract": { "schema_version": "1.0", @@ -66,22 +66,16 @@ "category": "external", "owner": "operator with provider credentials", "ledger_status": "pending", - "submission_status": "invalid-contract", - "intake_status": "fail", + "submission_status": "missing", + "intake_status": "missing", "source_accepted": true, - "review_state": "fix-submission", - "blocking_reason": "Submission exists but fails the ledger submission contract.", + "review_state": "awaiting-submission", + "blocking_reason": "No evidence packet has been submitted for review.", "submission_path": "evidence/world_class/submissions/provider-holdout.json", - "submitted_by": "Codex operator using DeepSeek official API", - "submitted_at": "2026-06-17", - "artifact_ref_count": 1, - "intake_errors": [ - "attestation.ledger_reviewer_approved must be true for a real submission", - "attestation.ledger_reviewer is required", - "attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ", - "attestation.ledger_reviewer must be different from submitted_by", - "attestation.ledger_reviewed_at must be at or after submitted_at" - ], + "submitted_by": "", + "submitted_at": "", + "artifact_ref_count": 0, + "intake_errors": [], "observed_state": { "model_executed_count": 10, "timing_observed_count": 10, diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 07503ec..5734e5d 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -4,13 +4,13 @@ Generated at: `2026-06-17` ## Summary -- decision: `fix-submissions` +- decision: `awaiting-submissions` - review items: `4` - accepted: `0` -- awaiting submission: `3` +- awaiting submission: `4` - valid packet but source incomplete: `0` - ready for ledger review: `0` -- fix submission: `1` +- fix submission: `0` - unmatched submissions: `0` - ready to claim world-class: `false` - review counts submission as completion: `false` @@ -21,7 +21,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w | Evidence | Review state | Intake | Source accepted | Submission | Next action | | --- | --- | --- | --- | --- | --- | -| `provider-holdout` | `fix-submission` | `fail` | `true` | `invalid-contract` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `provider-holdout` | `awaiting-submission` | `missing` | `true` | `missing` | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | `human-adjudication` | `awaiting-submission` | `missing` | `false` | `missing` | Record real A/B choices, reviewer metadata, and blind-review attestation, then regenerate adjudication. | | `native-permission-enforcement` | `awaiting-submission` | `missing` | `false` | `missing` | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | `native-client-telemetry` | `awaiting-submission` | `missing` | `false` | `missing` | Install a real client against the native host and import production metadata-only events. | @@ -30,11 +30,11 @@ This report is a read-only reviewer queue. It does not accept evidence or make w ### Provider Holdout -- review state: `fix-submission` -- blocking reason: Submission exists but fails the ledger submission contract. +- review state: `awaiting-submission` +- blocking reason: No evidence packet has been submitted for review. - ledger status: `pending` -- submission status: `invalid-contract` -- intake status: `fail` +- submission status: `missing` +- intake status: `missing` - source accepted: `true` - submission path: `evidence/world_class/submissions/provider-holdout.json` @@ -53,11 +53,7 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Intake Errors -- attestation.ledger_reviewer_approved must be true for a real submission -- attestation.ledger_reviewer is required -- attestation.ledger_reviewed_at must use YYYY-MM-DD or YYYY-MM-DDTHH:MM:SSZ -- attestation.ledger_reviewer must be different from submitted_by -- attestation.ledger_reviewed_at must be at or after submitted_at +- No intake errors. #### Privacy Contract