diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 23f973de..36ee29c5 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,24 +3,21 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "git_status": { "available": true, "dirty": true, - "changed_file_count": 36, + "changed_file_count": 9, "sample": [ - " M registry/index.json", - " M registry/packages/yao-meta-skill.json", - " M reports/adoption_drift_report.json", - " M reports/architecture_maintainability.json", - " M reports/architecture_maintainability.md", " M reports/benchmark_reproducibility.json", " M reports/benchmark_reproducibility.md", - " M reports/context_budget.json", - " M reports/context_budget.md", - " M reports/context_budget_summary.json", " M reports/evidence_consistency.json", - " M reports/install_simulation.json" + " M reports/review-studio.html", + " M reports/review-studio.json", + " M reports/review-viewer.json", + " M reports/skill-interpretation.json", + " M reports/skill-os-2-review.md", + " M reports/skill-overview.json" ], "scope": "generation-time status before this report is written" }, @@ -52,7 +49,7 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 36 + "changed_file_count": 9 }, "public_claim": { "ready": false, @@ -68,7 +65,7 @@ }, "release_lock": { "ready": false, - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "status_scope": "generation-time status before this report is written", "reason": "working tree was dirty at generation time" }, diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index 3d0b7310..f0c1a86b 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,7 +1,7 @@ # Benchmark Reproducibility Generated at: `2026-06-16` -Commit: `391821240ba14f088d0fbe12dd5f1fdca7917a39` +Commit: `a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf` Working tree dirty at generation: `true` Evidence bundle SHA256: `5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8` @@ -23,7 +23,7 @@ Evidence bundle SHA256: `5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24 - world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` - public claim blockers: `5` -- changed files at generation: `36` +- changed files at generation: `9` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index 59a2fd12..bb9f3623 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -47,8 +47,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": false, - "actual": false, + "expected": true, + "actual": true, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -58,8 +58,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "391821240ba14f088d0fbe12dd5f1fdca7917a39", - "actual": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "expected": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", + "actual": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -71,7 +71,7 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", @@ -81,10 +81,10 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 4 }, "actual": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", @@ -94,7 +94,7 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 4 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -204,8 +204,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "391821240ba14f088d0fbe12dd5f1fdca7917a39", - "actual": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "expected": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", + "actual": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -217,7 +217,7 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", @@ -227,10 +227,10 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 4 }, "actual": { - "release_lock_ready": false, + "release_lock_ready": true, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", @@ -240,7 +240,7 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 5 + "public_claim_blocker_count": 4 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -2025,9 +2025,9 @@ "label": "Skill OS 2.0 review summary mirrors current evidence", "status": "pass", "expected": [ - "score `89`", + "score `91`", "`16` gates", - "`4` warnings", + "`3` warnings", "`29` declared internal modules", "`82 / 82` CLI help smoke checks passing across `111` scripts", "`618` zip entries", diff --git a/reports/review-studio.html b/reports/review-studio.html index 698405fa..4c46c3c7 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -670,28 +670,28 @@
审查结论 review - Score 89/100 + Score 91/100

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

111 scripts scanned; secrets found

Py Compat0

177 files scanned for Python 3.11

Arch Debt0

899 largest lines; 8 watchlist; 64 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

173 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

618 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

4 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

111 scripts scanned; secrets found

Py Compat0

177 files scanned for Python 3.11

Arch Debt0

899 largest lines; 8 watchlist; 64 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

173 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

618 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
关注

上下文

initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 111 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 177 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

174 Python files; 0 hotspots; 8 watchlist files; 0 blockers; largest 899 lines; 64 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 2 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance governed; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 111 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 177 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

174 Python files; 0 hotspots; 8 watchlist files; 0 blockers; largest 899 lines; 64 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

阻断事项

无。

-

关注事项

+

关注事项

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

上下文

压缩或拆分高成本 deferred resources,保留最小可路由上下文。

初始加载可以安全,但后续 references、scripts、evals 体量过大时,reviewer 仍需要看到维护和读取成本。
修复位置
SKILL.md + references/ + scripts/ + evals/
验证命令
python3 scripts/render_context_reports.py
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据;提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  2. python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  3. Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时,Skill 只能证明会触发,不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间,或修掉 warning。

warning 可以被接受,但必须可审计、会过期,并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据,或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据;计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据;提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. YAO_OUTPUT_EVAL_MODEL=gpt-4.1-mini OPENAI_API_KEY=<redacted> python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  2. python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  3. Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
@@ -743,7 +743,7 @@
-

上下文

initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4

+

上下文

initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance governed; quality density 135.4

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

@@ -781,7 +781,7 @@

人工批准

warning 可以被有边界地接受,但必须写入 reviewer、理由、范围和到期时间;blocker 与 world-class 完成证据不能通过 waiver 变成通过。

-

批准概况

0 active waivers; 2 warning gates still need reviewer decision

+

批准概况

0 active waivers; 1 warning gates still need reviewer decision

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0

批准候选

@@ -806,8 +806,8 @@
-

公开声明

本地复现
发布锁
可公开声明
声明阻断
5
Provider 证据
人审完成
世界级就绪
-

声明阻断

+

公开声明

本地复现
发布锁
可公开声明
声明阻断
4
Provider 证据
人审完成
世界级就绪
+

声明阻断

diff --git a/reports/review-studio.json b/reports/review-studio.json index 931d7eb2..f4a28249 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -4,11 +4,11 @@ "skill_dir": ".", "summary": { "decision": "review", - "world_class_score": 89, + "world_class_score": 91, "gate_count": 16, "blocker_count": 0, - "warning_count": 4, - "action_count": 4, + "warning_count": 3, + "action_count": 3, "annotation_count": 0, "open_annotation_count": 0, "open_annotation_blocker_count": 0, @@ -42,8 +42,8 @@ { "key": "context-budget", "label": "上下文", - "status": "warn", - "detail": "initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4", + "status": "pass", + "detail": "initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance governed; quality density 135.4", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -115,7 +115,7 @@ "key": "review-waivers", "label": "人工批准", "status": "warn", - "detail": "0 active waivers; 2 warning gates still need reviewer decision", + "detail": "0 active waivers; 1 warning gates still need reviewer decision", "evidence": "reports/review_waivers.json", "link": "review_waivers.md" }, @@ -154,19 +154,11 @@ "evidence": "reports/output_quality_scorecard.json", "link": "output_quality_scorecard.md" }, - { - "key": "context-budget", - "label": "上下文", - "status": "warn", - "detail": "initial load 960/1000; deferred 436837/120000; top deferred scripts 387626; resource governance needs-review; quality density 135.4", - "evidence": "reports/context_budget.json", - "link": "context_budget.md" - }, { "key": "review-waivers", "label": "人工批准", "status": "warn", - "detail": "0 active waivers; 2 warning gates still need reviewer decision", + "detail": "0 active waivers; 1 warning gates still need reviewer decision", "evidence": "reports/review_waivers.json", "link": "review_waivers.md" }, @@ -243,53 +235,6 @@ "verification_command": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review", "evidence_steps": [] }, - { - "gate_key": "context-budget", - "label": "上下文", - "status": "warn", - "priority": "warning", - "summary": "压缩或拆分高成本 deferred resources,保留最小可路由上下文。", - "why": "初始加载可以安全,但后续 references、scripts、evals 体量过大时,reviewer 仍需要看到维护和读取成本。", - "source_fix": "SKILL.md + references/ + scripts/ + evals/", - "source_refs": [ - { - "path": "SKILL.md", - "label": "entrypoint", - "kind": "source", - "line": 9, - "exists": true, - "link": "../SKILL.md" - }, - { - "path": "reports/context_budget.md", - "label": "context budget", - "kind": "report", - "line": 1, - "exists": true, - "link": "context_budget.md" - }, - { - "path": "scripts/resource_boundary_check.py", - "label": "resource boundary checker", - "kind": "source", - "line": 44, - "exists": true, - "link": "../scripts/resource_boundary_check.py" - }, - { - "path": "references/skill-engineering-method.md", - "label": "skill engineering method", - "kind": "method", - "line": 208, - "exists": true, - "link": "../references/skill-engineering-method.md" - } - ], - "evidence": "reports/context_budget.json", - "evidence_link": "context_budget.md", - "verification_command": "python3 scripts/render_context_reports.py", - "evidence_steps": [] - }, { "gate_key": "review-waivers", "label": "人工批准", @@ -1731,13 +1676,13 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "d4d58b9557d03656a800aad0acee41d5434db006ebbbff2645fbbab80962cc27", - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "evidence_bundle_sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -1755,11 +1700,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 5, - "working_tree_dirty": true, - "changed_file_count": 20 + "public_claim_blocker_count": 4, + "working_tree_dirty": false, + "changed_file_count": 0 }, - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -4401,7 +4346,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.59, + "duration_ms": 26.79, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4429,7 +4374,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.1, + "duration_ms": 27.31, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4452,7 +4397,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.12, + "duration_ms": 26.66, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4480,7 +4425,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.36, + "duration_ms": 26.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4503,7 +4448,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.38, + "duration_ms": 26.62, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4531,7 +4476,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.48, + "duration_ms": 26.34, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4581,7 +4526,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.88, + "duration_ms": 26.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4604,7 +4549,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.99, + "duration_ms": 26.44, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4633,7 +4578,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.23, + "duration_ms": 26.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5354,36 +5299,23 @@ "ok": true, "generated_at": "2026-06-16", "skill_dir": ".", - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "git_status": { "available": true, - "dirty": true, - "changed_file_count": 20, - "sample": [ - " M reports/adoption_drift_report.json", - " M reports/architecture_maintainability.json", - " M reports/architecture_maintainability.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", - " M reports/context_budget.json", - " M reports/context_budget.md", - " M reports/context_budget_summary.json", - " M reports/output_execution_runs.json", - " M reports/output_execution_runs.md", - " M reports/python_compatibility.json", - " M reports/python_compatibility.md" - ], + "dirty": false, + "changed_file_count": 0, + "sample": [], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "d4d58b9557d03656a800aad0acee41d5434db006ebbbff2645fbbab80962cc27", - "source_contract_sha256": "f48a71e36bdbcaa353bd4e6bfc499bda41f047aeffe3d8eae2f99509ec693086", - "archive_sha256": "00151114ce7fee627882964137dd5259583f5521c75c6570789345e874454137", + "evidence_bundle_sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8", + "source_contract_sha256": "9fde13846a41c2d1442b6e80350313246dedebe02590ce1f2f07f2da1109c9ea", + "archive_sha256": "83ece575ef740a97b6a398f77ed4bc9f1869cde756257c64fdd90d8785c6f3a5", "output_case_count": 5, "failure_disclosure_count": 3, "command_count": 22, @@ -5401,15 +5333,14 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 5, - "working_tree_dirty": true, - "changed_file_count": 20 + "public_claim_blocker_count": 4, + "working_tree_dirty": false, + "changed_file_count": 0 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ - "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -5418,10 +5349,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": false, - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "ready": true, + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "status_scope": "generation-time status before this report is written", - "reason": "working tree was dirty at generation time" + "reason": "clean generation-time HEAD" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -5429,7 +5360,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "d4d58b9557d03656a800aad0acee41d5434db006ebbbff2645fbbab80962cc27" + "sha256": "5ad19434d41f67e6a8dbcbccbd22ffefed11008d75ebb00a78f0d24f3f8910a8" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5503,7 +5434,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "2ee3f1028f48673cddeb6242eb30aee3db1eccf26c44499744f3031d6d3520d7" + "sha256": "56430f01ba194dc6dc793eb505f92fec7a8c46e1be4b71976a3d641c9b98c7ec" }, { "label": "blind_review", @@ -5537,8 +5468,8 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 109099, - "sha256": "491354ae881f6682687a62a98c9a4988bbeef931b24565fdf43e6f3a206fd449" + "bytes": 109726, + "sha256": "a3a6d4e6809ea0b78b8012e37ad68903a54876a06100434efad38ea6bfe9b2bd" }, { "label": "python_compatibility", @@ -5552,21 +5483,21 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "870c455a5cd209ff8d9e8793861da074e668083d265230d04e38e8ee0ff1323a" + "sha256": "d6b523cc5cbb2d69c35989913e7e68dd4dc2bb04d3bcff88bbcb56fbe8be5084" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19338, - "sha256": "c4acb824482d992e0e4339b7eeeb8d23d49da12d3414606222f4916c3ed09df6" + "sha256": "ae8ac3872be3299b3d2dc18a29128346a3ecbb579987b1b08156e305237b5119" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, "bytes": 8604, - "sha256": "28ceb014e202269bb96628391d056234794b3570ff25de0ef9bdfd7f755fd85f" + "sha256": "3e27a6f7678efe9d94528f811a9d3acf20cf41d6328ad5b5fa0be5714c6396a0" }, { "label": "skill_os2_audit", @@ -6008,7 +5939,7 @@ "label": "Review Studio", "status": "pass", "objective": "One HTML page supports first-pass production review across trigger, output, runtime, trust, release, and evidence actions.", - "current": "16 gates; decision review; warnings 3", + "current": "16 gates; decision review; warnings 4", "command": "python3 scripts/yao.py review-studio .", "test": "python3 tests/verify_review_studio.py", "evidence": [ @@ -15488,16 +15419,14 @@ "context_budget": { "ok": true, "failures": [], - "warnings": [ - "Deferred resource footprint is high: 436837 estimated tokens across references/scripts/evals. Keep Review Studio warnings visible until the largest resource dirs are split, archived, or justified." - ], + "warnings": [], "stats": { "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1376303, + "other_text_tokens": 1377236, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1377070, + "estimated_total_text_tokens": 1378003, "deferred_resource_tokens": 436837, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ @@ -15525,31 +15454,30 @@ } ], "deferred_resource_governance": { - "status": "needs-review", + "status": "governed", "large_dir_count": 1, - "governed_large_dir_count": 0, + "governed_large_dir_count": 1, "directories": [ { - "status": "needs-review", + "status": "governed", "evidence": [ "reports/security_trust_report.json", "reports/architecture_maintainability.json", "reports/python_compatibility.json" ], "reasons": [ + "trust report covers scripts", "architecture report has no script hotspots or blockers", "Python compatibility report has no issues" ], - "missing": [ - "trust report covers scripts" - ], + "missing": [], "path": "scripts", "estimated_tokens": 387626, "file_count": 111, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], - "summary": "One or more large deferred resource directories still need explicit governance evidence." + "summary": "Large deferred resources are indexed and backed by evidence." }, "relevant_file_count": 565, "unused_resource_dirs": [], @@ -17419,7 +17347,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-15T17:03:40Z", + "generated_at": "2026-06-15T17:08:05Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/review-viewer.json b/reports/review-viewer.json index c48a2612..e18272db 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -996,7 +996,7 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": false, + "release_lock_ready": true, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, @@ -1020,11 +1020,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 5, - "working_tree_dirty": true, - "changed_file_count": 36 + "public_claim_blocker_count": 4, + "working_tree_dirty": false, + "changed_file_count": 0 }, - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index a3a7a072..40282d39 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -1026,9 +1026,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 36 + "changed_file_count": 9 }, - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 71d9e581..f4d46109 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -37,7 +37,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Local Install Sync Preflight v0 so source-to-local and source-to-active install syncs rebuild the package first, run install simulation, enforce installer permission coverage, and refuse to copy files before any destructive sync when the package is not install-ready. - Atlas Scope Policy v0 so examples, evolution snapshots, embedded generated skills, and validator fixtures remain visible in the full portfolio report without polluting release-actionable gates. - Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision. -- Review Studio now avoids over-claiming release readiness when blind A/B adjudication, context-budget pressure, waiver handling, and world-class evidence are still pending: the root Meta Skill is in `review` with score `89`, no blockers, four warnings, and explicit actions for Output Lab reviewer adjudication, context-budget follow-up, waiver handling, and world-class evidence completion. +- Review Studio now avoids over-claiming release readiness when blind A/B adjudication, waiver handling, and world-class evidence are still pending: the root Meta Skill is in `review` with score `91`, no blockers, three warnings, and explicit actions for Output Lab reviewer adjudication, waiver handling, and world-class evidence completion. - Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning. - Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case. - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. @@ -91,7 +91,7 @@ Next move: run provider-backed holdout cases with real credentials and one real ### 3. Review Studio is unified and now has reviewer actions plus annotations -The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, world-class evidence, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. The current root report is intentionally not a clean pass: decision `review`, score `89`, `16` gates, `0` blockers, `4` warnings, `4` actions, and `0` open annotation blockers. The warnings are useful evidence: the automated Output Eval Lab is strong, but the blind A/B adjudication still has `5` pending reviewer decisions, the context budget is close to its limit, and the world-class ledger still has `4` pending external or human evidence entries, so the package should not claim fully reviewed or world-class status. +The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, world-class evidence, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. The current root report is intentionally not a clean pass: decision `review`, score `91`, `16` gates, `0` blockers, `3` warnings, `3` actions, and `0` open annotation blockers. The warnings are useful evidence: the automated Output Eval Lab is strong, but the blind A/B adjudication still has `5` pending reviewer decisions and the world-class ledger still has `4` pending external or human evidence entries, so the package should not claim fully reviewed or world-class status. Next move: add richer source-line anchors inside generated reports, record real reviewer annotations during the next human review pass, and close the provider, native-permission, real-client telemetry, and human review evidence entries in the world-class ledger. @@ -126,7 +126,7 @@ Next move: add real client or installer permission enforcement integration. | Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | | Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0, and world-class evidence gaps must be closed with accepted evidence rather than waived as completed | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | -| Review Studio | decision `review`, world-class score `89`, `16` gates, `0` blockers, `4` warnings, `4` review actions, `0` open annotation blockers | +| Review Studio | decision `review`, world-class score `91`, `16` gates, `0` blockers, `3` warnings, `3` review actions, `0` open annotation blockers | | Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps | | World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts | | World-Class Evidence Ledger | `4` evidence entries remain pending: `1` human-required and `3` external-required; anti-overclaim guards block planned work, metadata fallbacks, pending review, and local command runners from counting as completion | diff --git a/reports/skill-overview.json b/reports/skill-overview.json index d66178e6..93e32078 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -1021,9 +1021,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 36 + "changed_file_count": 9 }, - "commit": "391821240ba14f088d0fbe12dd5f1fdca7917a39", + "commit": "a7960d4cfd2d9a17cbaebcee7b823f0db13df0cf", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",