diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index f9d05869..2fdf79fa 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "python_file_count": 215, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 421450a2..acade6a2 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -1,6 +1,6 @@ # Architecture Maintainability -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/context_budget.json b/reports/context_budget.json index 8b2a36ec..64bb91e9 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1076786, + "other_text_tokens": 1078122, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1077583, - "deferred_resource_tokens": 495216, + "estimated_total_text_tokens": 1078919, + "deferred_resource_tokens": 495718, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140 }, { @@ -36,7 +36,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140 } ], @@ -59,7 +59,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/context_budget.md b/reports/context_budget.md index e15688a3..7627a4c4 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 495216 | `governed` | scripts:435118 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 495718 | `governed` | scripts:435620 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index f26e9312..d2d7a29f 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 495216, + "deferred_resource_tokens": 495718, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 207c59b9..1c479960 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "root": ".", "summary": { "target_python": "3.11", diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 1289f5d7..03a8bf61 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -1,6 +1,6 @@ # Python Compatibility -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/review-studio.html b/reports/review-studio.html index 42dfb23a..9893aed6 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -740,12 +740,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint21/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

140 scripts scanned; secrets found

Py Compat0

218 files scanned for Python 3.11

Arch Debt0

696 largest lines; 0 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

182 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

658 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

5 blockers; local reproducible true

Blueprint20/21

2.0 coverage; extensions partial 0, planned 0; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

140 scripts scanned; secrets found

Py Compat0

218 files scanned for Python 3.11

Arch Debt0

696 largest lines; 0 watchlist; 68 CLI handlers; 18 in entrypoint

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Daily Ops5

proposal-review; approval 0; release lock true

Weekly Queue5

curator-review; ready 1; top score 88

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

182 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

658 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 495216/120000; top deferred scripts 435118; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 140 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 218 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

215 Python files; 0 hotspots; 0 watchlist files; 0 blockers; largest 696 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 990/1000; deferred 495718/120000; top deferred scripts 435620; resource governance governed; quality density 131.3

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 140 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 218 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

215 Python files; 0 hotspots; 0 watchlist files; 0 blockers; largest 696 lines; 68 CLI handlers; 18 in entrypoint

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -755,12 +755,12 @@

阻断事项

无。

-

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true
+

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision
  • 世界证据4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  2. export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  3. python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 0

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
2 blocked / 1 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_operator_runbook.html + reports/world_class_evidence_ledger.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-runbook . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py review-studio .

证据采集

以下条目仍需真实外部或人工证据；提交文件、校验命令和阻断检查必须同时闭环。

pending · external

Provider Holdout

model-executed 0; token-observed 0

提交
evidence/world_class/submissions/provider-holdout.json
模板
evidence/world_class/templates/provider-holdout.intake.json
阻断
2 blocked / 1 pass
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
阻断检查
  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key provider-holdout --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  2. export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  3. python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
采集契约
来源要求
  • provider-backed model run
  • observed timing
  • observed token metadata
通过条件
  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass
证据资产
  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref1 / 1 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
pending · human

Human Adjudication

0/5 decisions; pending 5

提交
evidence/world_class/submissions/human-adjudication.json
模板
evidence/world_class/templates/human-adjudication.intake.json
阻断
2 blocked / 2 pass
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
阻断检查
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key human-adjudication --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/yao.py output-review-kit --write-template
  2. Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  3. python3 scripts/adjudicate_output_review.py --write-template
采集契约
来源要求
  • real reviewer identity
  • blind A/B decisions
  • answer key unopened until decisions exist
通过条件
  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass
证据资产
  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence8 / 8 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.
pending · external

Native Permission Enforcement

native-enforced targets 0; installer-enforced targets 4

提交
evidence/world_class/submissions/native-permission-enforcement.json
模板
evidence/world_class/templates/native-permission-enforcement.intake.json
阻断
1 blocked / 2 pass
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
阻断检查
  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-permission-enforcement --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  2. Update the generated target adapter only when the guard is actually enforced by that target.
  3. python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
采集契约
来源要求
  • real target client or external installer runtime guard
  • native enforcement flag or externally accepted guard proof
  • residual risk retained for fallback targets
通过条件
  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
证据资产
  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence11 / 11 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
pending · external

Native Client Telemetry

external source events 0; adoption samples 1

提交
evidence/world_class/submissions/native-client-telemetry.json
模板
evidence/world_class/templates/native-client-telemetry.intake.json
阻断
1 blocked / 2 pass
下一步
Install a real client against the native host and import production metadata-only events.
阻断检查
  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
操作命令
  • 准备提交python3 scripts/yao.py world-class-submission-kit . --evidence-key native-client-telemetry --output-dir evidence/world_class/submissions
  • 校验入口python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions
  • 审查提交python3 scripts/yao.py world-class-submission-review . --submissions-dir evidence/world_class/submissions
  • 刷新台账python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions
首要步骤
  1. python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  2. Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  3. python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
采集契约
来源要求
  • real external client source
  • metadata-only event
  • local-first import path
通过条件
  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass
证据资产
  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
资产角色

source: world-class-submission-kit; counts as evidence: false; prefill counts as evidence: false

  • submission-ref2 / 2 readyartifact_refs: trueRows marked submission-ref are the aggregate paths expected in artifact_refs.
  • supporting-evidence6 / 6 readyartifact_refs: falseSupporting-evidence rows help reviewers audit the packet but do not all need to be copied into artifact_refs.
隐私边界
  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -812,12 +812,12 @@
-

上下文

initial load 990/1000; deferred 495216/120000; top deferred scripts 435118; resource governance governed; quality density 131.3

+

上下文

initial load 990/1000; deferred 495718/120000; top deferred scripts 435620; resource governance governed; quality density 131.3

编译证据

Review reports/compiled_targets.md before packaging to inspect target adapter modes, generated files, preserved semantics, warnings, and unsupported features.

-

信任报告

Secret
0
脚本数
140
网络脚本
3
Help 失败
0
包体哈希
d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69
+

信任报告

Secret
0
脚本数
140
网络脚本
3
Help 失败
0
包体哈希
9cb9a19e774540e721b143a240baf41b41c32b9f10df532eb4a7d1cf7a2fe67b

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -842,8 +842,8 @@
-

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

-

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true

+

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
@@ -861,13 +861,13 @@

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0

批准候选

-
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 0; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
无
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .
+
可批准 · needs-reviewer-decision

Output Lab

review pending 5; model-executed 0; output failures 0

Gate
output-lab
证据
reports/output_review_adjudication.md
选项
accepted-risk, false-positive, temporary-exception
边界
Does not count as provider, human, or public world-class completion evidence.

审查条件

  • Reviewer confirms this release does not claim provider-backed or human-adjudicated output superiority.
  • Reviewer names the release scope and expiry date.
  • Reviewer links output_review_adjudication or output_execution evidence.
建议命令python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer "<reviewer>" --reason "Output Lab has pending human/provider evidence; accepted only for this bounded review scope." --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md
不可批准 · cannot-waive

World-Class Evidence

4 pending evidence entries; 1 human pending; 3 external pending

Gate
world-class-evidence
证据
reports/world_class_evidence_ledger.md
选项
无
边界
Non-waivable completion boundary.

审查条件

  • Do not use a waiver to claim public world-class readiness.
  • Either submit accepted ledger evidence or state that this release does not claim world-class completion.
  • Keep claim guard active until ledger summary.ready_to_claim_world_class is true.
建议命令python3 scripts/yao.py world-class-ledger . --submissions-dir evidence/world_class/submissions && python3 scripts/yao.py world-class-claim-guard .

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Set OPENAI_API_KEY in the operator shell before running provider evidence; never commit or print the value.
  • export YAO_OUTPUT_EVAL_MODEL=${YAO_OUTPUT_EVAL_MODEL:-gpt-4.1-mini}
  • python3 scripts/yao.py output-exec --provider-runner openai --timeout-seconds 60
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/provider-holdout.intake.json to evidence/world_class/submissions/provider-holdout.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/yao.py output-review-kit --write-template
  • Open reports/output_review_kit.md and choose A or B for each pair without opening the answer key.
  • python3 scripts/adjudicate_output_review.py --write-template
  • Record reviewer choices in a separate JSON, JSONL, or CSV decision source with case_id, winner_variant, confidence, and reason only.
  • python3 scripts/yao.py output-review-import --input <reviewer-decisions.json> --run-adjudication
  • python3 scripts/yao.py output-review
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/human-adjudication.intake.json to evidence/world_class/submissions/human-adjudication.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • scripts/import_output_review_decisions.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • The decision importer rejects raw prompt, output, transcript, message, and answer-key fields.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • Implement or connect a real target client or external installer runtime guard that blocks undeclared network, file_write, or subprocess capabilities.
  • Update the generated target adapter only when the guard is actually enforced by that target.
  • python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip
  • python3 scripts/yao.py install-simulate . --package-dir dist --install-root dist/install-simulation
  • python3 scripts/yao.py runtime-permissions . --package-dir dist
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-permission-enforcement.intake.json to evidence/world_class/submissions/native-permission-enforcement.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 1
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 1; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

执行步骤

  • python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/
  • Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.
  • python3 scripts/yao.py telemetry-import . --input-jsonl .yao/telemetry_spool/external_events.jsonl
  • python3 scripts/yao.py skill-atlas --workspace-root .
  • python3 scripts/yao.py skill-os2-audit . --generated-at <YYYY-MM-DD>
  • Copy evidence/world_class/templates/native-client-telemetry.intake.json to evidence/world_class/submissions/native-client-telemetry.json and fill only real evidence fields.
  • python3 scripts/yao.py world-class-intake . --submissions-dir evidence/world_class/submissions

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 1 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
@@ -877,17 +877,17 @@
-

蓝图覆盖

项目数
21
模块数
8
建议 PR
13
通过数
21
Warn Count
0
缺失数
0
Extension Track Count
4
Extension Covered Count
4
Extension Partial Count
0
Extension Planned Count
0
Adaptive Extension Ready
是
本地蓝图
是
世界级
否
待补证据
4
+

蓝图覆盖

项目数
21
模块数
8
建议 PR
13
通过数
20
Warn Count
1
缺失数
0
Extension Track Count
4
Extension Covered Count
4
Extension Partial Count
0
Extension Planned Count
0
Adaptive Extension Ready
是
本地蓝图
否
世界级
否
待补证据
4

覆盖边界

蓝图覆盖只证明 2.0 模块、建议 PR、脚本、报告和测试在本地闭环；public world-class 仍以 world-class evidence ledger 的真人和外部证据为准。

公开声明

本地复现
是
发布锁
否
可公开声明
否
声明阻断
5
Provider 证据
否
人审完成
否
世界级就绪
否
-

声明阻断

+

声明阻断

-

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true

+

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true

证据台账

Ledger Entry Count
4
Accepted Count
0
待审
4
Human Pending Count
1
External Pending Count
3
Overclaim Guard Active
是
Ready To Claim World Class
否
diff --git a/reports/review-studio.json b/reports/review-studio.json index ad952ca1..056b9460 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -44,7 +44,7 @@ "key": "context-budget", "label": "上下文", "status": "pass", - "detail": "initial load 990/1000; deferred 495216/120000; top deferred scripts 435118; resource governance governed; quality density 131.3", + "detail": "initial load 990/1000; deferred 495718/120000; top deferred scripts 435620; resource governance governed; quality density 131.3", "evidence": "reports/context_budget.json", "link": "context_budget.md" }, @@ -108,7 +108,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", + "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low; daily proposals 5; daily decision proposal-review; daily release lock true; weekly queue 5 unique; weekly ready 1; weekly top 88; weekly release lock true", "evidence": "reports/adoption_drift_report.json + reports/skillops/daily + reports/skillops/weekly", "link": "adoption_drift_report.md" }, @@ -124,7 +124,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md" }, @@ -212,7 +212,7 @@ "key": "world-class-evidence", "label": "世界证据", "status": "warn", - "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 6/13 pass; 7 blocked; overclaim guard true", + "detail": "4 pending world-class evidence entries; 1 human pending; 3 external pending; source checks 7/13 pass; 6 blocked; overclaim guard true", "evidence": "reports/world_class_evidence_ledger.json", "link": "world_class_evidence_ledger.md" } @@ -817,12 +817,12 @@ "category": "external", "status": "pending", "readiness": "awaiting-submission", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "next_action": "Install a real client against the native host and import production metadata-only events.", "submission_path": "evidence/world_class/submissions/native-client-telemetry.json", "template_path": "evidence/world_class/templates/native-client-telemetry.intake.json", - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "blocked_checks": [ { "label": "External events", @@ -831,14 +831,6 @@ "expected": ">0", "status": "blocked", "next_action": "Import at least one metadata-only event from a real client." - }, - { - "label": "Adoption sample", - "field": "adoption_sample_count", - "actual": 0, - "expected": ">0", - "status": "blocked", - "next_action": "Telemetry must include adoption outcome evidence." } ], "commands": [ @@ -1383,7 +1375,7 @@ "external_pending_count": 3, "human_pending_count": 1, "source_check_count": 13, - "source_pass_count": 6, + "source_pass_count": 7, "conclusion_zh": "世界级证据尚未完成：4 项待补，0 项已接受。", "conclusion_en": "World-class evidence is not complete: 4 pending, 0 accepted.", "entries": [ @@ -1442,8 +1434,7 @@ "summary_zh": "真实外部客户端 metadata-only 事件仍未导入。", "summary_en": "Real external-client metadata-only events have not been imported yet.", "blocked_checks": [ - "External events", - "Adoption sample" + "External events" ] } ] @@ -1972,8 +1963,8 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "678c3839e1d41692b4fb991e542e7d6ef60fbca5248dab2e3f313e138aedde77", - "source_contract_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69", + "evidence_bundle_sha256": "c31f7210165c76ab9b288c6c7d5d228fd46c8c8627cd86ca095cc1033607e34f", + "source_contract_sha256": "9cb9a19e774540e721b143a240baf41b41c32b9f10df532eb4a7d1cf7a2fe67b", "archive_sha256": "5802e5f522558cf77666ae59390a28845cf3affd22fa1267b6629891ddeba76b", "output_case_count": 5, "failure_disclosure_count": 3, @@ -1989,14 +1980,14 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 18 + "changed_file_count": 33 }, - "commit": "3961b60341a893464c1ea96f9921f804809f37f9", + "commit": "108f4d8b8736c5cda3be52697b44efaf4e785b42", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", @@ -2101,7 +2092,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 231, - "package_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69" + "package_sha256": "9cb9a19e774540e721b143a240baf41b41c32b9f10df532eb4a7d1cf7a2fe67b" }, "skill_atlas": { "skill_count": 12, @@ -2139,7 +2130,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "a163b6e288c5342e683875b47fdfe62989d3f09241c0c8fd86ae615c6d5da7ad", + "package_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69", "archive_sha256": "5802e5f522558cf77666ae59390a28845cf3affd22fa1267b6629891ddeba76b" }, "compatibility": { @@ -2176,7 +2167,7 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-17" }, "failures": [], "warnings": [] @@ -2199,7 +2190,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 658, + "archive_entry_count": 677, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -2271,7 +2262,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "a163b6e288c5342e683875b47fdfe62989d3f09241c0c8fd86ae615c6d5da7ad" + "to": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69" } ] }, @@ -2288,14 +2279,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -2304,7 +2295,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -2571,7 +2562,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2624,8 +2615,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -2975,7 +2966,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -3012,7 +3003,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -3033,8 +3024,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -3050,8 +3041,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -4701,7 +4692,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.96, + "duration_ms": 26.12, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4729,7 +4720,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.2, + "duration_ms": 28.28, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4752,7 +4743,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.36, + "duration_ms": 29.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4780,7 +4771,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.09, + "duration_ms": 29.8, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4803,7 +4794,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.43, + "duration_ms": 29.59, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4831,7 +4822,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.16, + "duration_ms": 28.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4854,7 +4845,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.55, + "duration_ms": 26.89, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4881,7 +4872,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.86, + "duration_ms": 26.9, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4904,7 +4895,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.87, + "duration_ms": 26.03, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4933,7 +4924,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.17, + "duration_ms": 26.51, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5659,24 +5650,24 @@ "ok": true, "generated_at": "2026-06-17", "skill_dir": ".", - "commit": "3961b60341a893464c1ea96f9921f804809f37f9", + "commit": "108f4d8b8736c5cda3be52697b44efaf4e785b42", "git_status": { "available": true, "dirty": true, - "changed_file_count": 18, + "changed_file_count": 33, "sample": [ - " M reports/adaptation_proposals.json", - " M reports/adaptation_proposals.md", - " M reports/benchmark_reproducibility.json", - " M reports/benchmark_reproducibility.md", + " M reports/architecture_maintainability.json", + " M reports/architecture_maintainability.md", " M reports/context_budget.json", + " M reports/context_budget.md", " M reports/context_budget_summary.json", " M reports/evidence_consistency.json", + " M reports/evidence_consistency.md", " M reports/python_compatibility.json", " M reports/python_compatibility.md", " M reports/review-studio.html", " M reports/review-studio.json", - " M reports/review-viewer.json" + " M reports/security_trust_report.json" ], "scope": "generation-time status before this report is written" }, @@ -5686,8 +5677,8 @@ "methodology_complete": true, "required_artifact_count": 25, "missing_artifact_count": 0, - "evidence_bundle_sha256": "678c3839e1d41692b4fb991e542e7d6ef60fbca5248dab2e3f313e138aedde77", - "source_contract_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69", + "evidence_bundle_sha256": "c31f7210165c76ab9b288c6c7d5d228fd46c8c8627cd86ca095cc1033607e34f", + "source_contract_sha256": "9cb9a19e774540e721b143a240baf41b41c32b9f10df532eb4a7d1cf7a2fe67b", "archive_sha256": "5802e5f522558cf77666ae59390a28845cf3affd22fa1267b6629891ddeba76b", "output_case_count": 5, "failure_disclosure_count": 3, @@ -5703,12 +5694,12 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "world_class_source_check_count": 13, - "world_class_source_pass_count": 6, - "world_class_source_blocked_count": 7, + "world_class_source_pass_count": 7, + "world_class_source_blocked_count": 6, "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 18 + "changed_file_count": 33 }, "public_claim": { "ready": false, @@ -5718,13 +5709,13 @@ "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", - "world-class source checks are not all accepted (6/13 pass, 7 blocked)" + "world-class source checks are not all accepted (7/13 pass, 6 blocked)" ], "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { "ready": false, - "commit": "3961b60341a893464c1ea96f9921f804809f37f9", + "commit": "108f4d8b8736c5cda3be52697b44efaf4e785b42", "status_scope": "generation-time status before this report is written", "reason": "working tree was dirty at generation time" }, @@ -5734,7 +5725,7 @@ "existing_count": 25, "missing_count": 0, "missing_paths": [], - "sha256": "678c3839e1d41692b4fb991e542e7d6ef60fbca5248dab2e3f313e138aedde77" + "sha256": "c31f7210165c76ab9b288c6c7d5d228fd46c8c8627cd86ca095cc1033607e34f" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -5807,8 +5798,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7966, - "sha256": "2c9409158a128e6d2ad4c96c499dd412591c7b287d907e8d88987132a0592633" + "bytes": 7965, + "sha256": "6334c2f0261c2c978ebcb518cdac4ddf409f41e3732b08998c8e4b80529e446d" }, { "label": "blind_review", @@ -5842,22 +5833,22 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 129520, - "sha256": "0d60cae961055a68ac84d83cc23d6d8456cc4b42b1a103dd963f3343a20987ff" + "bytes": 129571, + "sha256": "5b05d1110b96a6a72300cc5913140a9d4c0bdf099b1232df8c3bce6e2d023514" }, { "label": "python_compatibility", "path": "reports/python_compatibility.json", "exists": true, "bytes": 28298, - "sha256": "b223212ec521b782c479ace75a0e93866e09aff7f2637fdd181e8906bb4b0376" + "sha256": "8b48340618cc702a8d8f5aaf6ce6d05a1a5463a8008d06a28809671632b212da" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "e75a341d15e4d89d46bded0cdb5c1ba378b5674df84d9182eef57c5fb6204c9e" + "sha256": "dfd07c1825de99435bd661e2663ae74251bbe03cb5ddda66d268efb6a872eb31" }, { "label": "package_verification", @@ -5870,71 +5861,71 @@ "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, - "bytes": 8604, - "sha256": "e29ae26bc97c94ec0a9da3ab60f28ef90e23c9d0c53b777810165224e725b452" + "bytes": 8758, + "sha256": "5269d6e6364aaa39e935cbf02df7569a6a88d2653120bc6e3e34d3684a73639d" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 14310, - "sha256": "f31dc78072fe38192f25148e11fd902c902e883e230585314172dcf1a3241f32" + "sha256": "390d0bb42652e9f84fcab195de1da3c9cda213c254fd9319d31c54dd8429aee7" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, "bytes": 20828, - "sha256": "0a836a86ad6c648d8c442a16cae7cdb17d3640a0ba6ec69c1290825ede6d152a" + "sha256": "9b74155fb9bfe44ba838d64b1a31768b83978b154bdd8eaa606f494c816f189f" }, { "label": "world_class_evidence_ledger", "path": "reports/world_class_evidence_ledger.json", "exists": true, - "bytes": 22288, - "sha256": "fe81833eab69292525975b347177bc7fcda6410f493fc60bd840a20e86ca0039" + "bytes": 22285, + "sha256": "f6e1a69b0359c28c88e9e9e69ff66763c721ac8a112a43a673ce98cd9d3bce41" }, { "label": "world_class_evidence_intake", "path": "reports/world_class_evidence_intake.json", "exists": true, "bytes": 19319, - "sha256": "4506c836338c65a3a01b15b1f26a1fdd0a8ef520a1e5f1574ee89ce9d3bbeb29" + "sha256": "46f801051ec075725059e4e2682ef2162fd892b5e7255c8d5121d7b24dcc7f2c" }, { "label": "world_class_evidence_preflight", "path": "reports/world_class_evidence_preflight.json", "exists": true, - "bytes": 41208, - "sha256": "725d10a6f951ab1e64f4e6a1cfa907404b51f598a753b5a0351e1381fef7cd9e" + "bytes": 41202, + "sha256": "b7fd72d7e75486612f65a9bb73b1ab59106103ebf3ad10dd446cd80903088454" }, { "label": "world_class_submission_review", "path": "reports/world_class_submission_review.json", "exists": true, - "bytes": 13658, - "sha256": "7693638c2e8369a8a179b3420c11090668f8f104f452dc370082dc9ebafe882d" + "bytes": 13655, + "sha256": "9fbd4cd437986fca9a043cbab24aec7d575aaf166a8d9365c79c56f093cab5b8" }, { "label": "world_class_operator_runbook", "path": "reports/world_class_operator_runbook.json", "exists": true, - "bytes": 24899, - "sha256": "f90d59c3e5866da33138869c1c5299350a43389d81e84ba466df794befb494fa" + "bytes": 24835, + "sha256": "2d32a2e917b7215afe7a7a93ca8bacffa540a245953aa39e2ee47eea1b169d17" }, { "label": "world_class_operator_runbook_markdown", "path": "reports/world_class_operator_runbook.md", "exists": true, - "bytes": 15479, - "sha256": "be79ee0f70a3ceed473c481562c5fddaee6cfb36c960ba15ae10c846d5809650" + "bytes": 15424, + "sha256": "00d8ebabb91e14a06cacdb212f448668aa1e47b72587448e30681fc53acaad9a" }, { "label": "world_class_operator_runbook_html", "path": "reports/world_class_operator_runbook.html", "exists": true, - "bytes": 21409, - "sha256": "9a7be02a6990245c9dd1f72ced5928d80a3aaebea4647737c2aaf0b0a2f798df" + "bytes": 21348, + "sha256": "ca72c65b437b2211d08b9d463c9a33a7993b4e388bb75f62757ead4dfcf81a49" }, { "label": "world_class_claim_guard", @@ -6087,22 +6078,22 @@ "item_count": 21, "module_count": 8, "recommended_pr_count": 13, - "pass_count": 21, - "warn_count": 0, + "pass_count": 20, + "warn_count": 1, "missing_count": 0, "extension_track_count": 4, "extension_partial_count": 0, "extension_planned_count": 0, "extension_covered_count": 4, "adaptive_extension_ready": true, - "local_blueprint_ready": true, + "local_blueprint_ready": false, "public_world_class_ready": false, "world_class_evidence_pending_count": 4, - "decision": "local-blueprint-covered-evidence-pending" + "decision": "continue-implementation" }, "status_counts": { - "pass": 21, - "warn": 0, + "pass": 20, + "warn": 1, "missing": 0 }, "extension_status_counts": { @@ -6691,9 +6682,9 @@ "key": "evidence-consistency", "category": "recommended-pr", "label": "Evidence Consistency", - "status": "pass", + "status": "warn", "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", - "current": "35 consistency checks", + "current": "36 consistency checks", "command": "make ci-test", "test": "tests/verify_evidence_consistency.py", "evidence": [ @@ -6892,7 +6883,7 @@ "compiled_targets": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-17", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "target_count": 5, @@ -11706,7 +11697,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "dist", + "package_dir": "tests/tmp_review_studio/dist", "expected_capabilities": [ "file_write", "network", @@ -11746,7 +11737,7 @@ { "target": "openai", "status": "pass", - "adapter": "dist/targets/openai/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11843,7 +11834,7 @@ { "target": "claude", "status": "pass", - "adapter": "dist/targets/claude/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -11925,7 +11916,7 @@ { "target": "generic", "status": "pass", - "adapter": "dist/targets/generic/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -12007,7 +11998,7 @@ { "target": "vscode", "status": "pass", - "adapter": "dist/targets/vscode/adapter.json", + "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -12118,7 +12109,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 231, - "package_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69" + "package_sha256": "9cb9a19e774540e721b143a240baf41b41c32b9f10df532eb4a7d1cf7a2fe67b" }, "failures": [], "warnings": [], @@ -12875,7 +12866,7 @@ "uses_input": false, "uses_network": false, "uses_file_write": true, - "uses_subprocess": false, + "uses_subprocess": true, "network_urls": [], "network_hosts": [] }, @@ -15370,6 +15361,7 @@ "scripts/ci_test.py", "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", @@ -15408,7 +15400,7 @@ "python_compatibility": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "root": ".", "summary": { "target_python": "3.11", @@ -16869,15 +16861,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1076737, + "other_text_tokens": 1078122, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1077534, - "deferred_resource_tokens": 495216, + "estimated_total_text_tokens": 1078919, + "deferred_resource_tokens": 495718, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140 }, { @@ -16899,7 +16891,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140 } ], @@ -16922,7 +16914,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 435118, + "estimated_tokens": 435620, "file_count": 140, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } @@ -17773,7 +17765,7 @@ }, "catalog": { "workspace_root": ".", - "generated_at": "2026-06-16", + "generated_at": "2026-06-17", "skills": [ { "name": "yao-meta-skill", @@ -17793,6 +17785,8 @@ "vscode" ], "resources": [ + "scripts/adaptation_patch_safety.py", + "scripts/adaptation_report_contracts.py", "scripts/adjudicate_output_review.py", "scripts/apply_adaptation.py", "scripts/build_confusion_matrix.py", @@ -17825,6 +17819,7 @@ "scripts/lint_skill.py", "scripts/local_output_eval_runner.py", "scripts/optimize_description.py", + "scripts/output_review_privacy.py", "scripts/prepare_output_review_kit.py", "scripts/prepare_world_class_submission_kit.py", "scripts/probe_runtime_permissions.py", @@ -17877,6 +17872,7 @@ "scripts/review_studio_actions.py", "scripts/review_studio_data.py", "scripts/review_studio_formatting.py", + "scripts/review_studio_gate_contract.py", "scripts/review_studio_gates.py", "scripts/review_studio_layout.py", "scripts/review_studio_output_review.py", @@ -17904,11 +17900,16 @@ "scripts/telemetry_native_host.py", "scripts/trigger_eval.py", "scripts/trust_check.py", + "scripts/trust_check_scripts.py", "scripts/upgrade_check.py", "scripts/validate_skill.py", "scripts/verify_package.py", "scripts/world_class_evidence_contract.py", + "scripts/world_class_human_evidence.py", + "scripts/world_class_native_permission_evidence.py", + "scripts/world_class_native_telemetry_evidence.py", "scripts/world_class_preflight_layout.py", + "scripts/world_class_provider_evidence.py", "scripts/world_class_source_checks.py", "scripts/world_class_submission_kit_rendering.py", "scripts/world_class_submission_matrix.py", @@ -17993,8 +17994,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, @@ -18657,7 +18658,7 @@ "name": "incident-command-governor", "path": "examples/governed-incident-command/generated-skill", "reason": "review overdue by cadence monthly", - "age_days": 77, + "age_days": 78, "allowed_days": 31, "actionable": false, "scope": "example" @@ -18825,7 +18826,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16T16:14:10Z", + "generated_at": "2026-06-16T16:15:39Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -18849,14 +18850,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -18865,7 +18866,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -18877,31 +18878,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 0, - "accepted": 0, + "adoption_events": 1, + "accepted": 1, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 0 + "adoption_rate": 100.0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "review_event", + "event": "skill_activation", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "manual", - "outcome": "reviewed", + "activation_type": "explicit", + "outcome": "accepted", "failure_type": "none", - "timestamp": "2026-06-13T12:00:00Z" + "timestamp": "2026-06-13T10:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "reports/telemetry_events.jsonl", + "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -19745,7 +19746,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-17", + "generated_at": "2026-06-13", "summary": { "waiver_count": 0, "active_count": 0, @@ -19835,7 +19836,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-17 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { @@ -19865,7 +19866,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "reports/review_annotations_input.json", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -20301,8 +20302,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -20318,8 +20319,8 @@ "missing_submission_count": 4, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "submitted_but_pending_count": 0, "source_accepted_without_valid_submission_count": 0, "overclaim_guard_active": true, @@ -20695,7 +20696,7 @@ "status": "pending", "source_status": "external_required", "source_accepted": false, - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -20732,7 +20733,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -20753,8 +20754,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -20770,8 +20771,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -21123,7 +21124,7 @@ "source_accepted": false, "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -21207,8 +21208,8 @@ "collection_ready_count": 1, "collection_blocked_count": 3, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "pending_count": 4, "ready_to_claim_world_class": false, "credential_value_exposed": false, @@ -21728,8 +21729,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -22084,8 +22085,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -22168,8 +22169,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "review_counts_submission_as_completion": false, "summary": { "review_item_count": 4, @@ -22181,8 +22182,8 @@ "unmatched_submission_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -22456,7 +22457,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -22477,8 +22478,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -22494,8 +22495,8 @@ } ], "source_check_count": 3, - "source_pass_count": 1, - "source_blocked_count": 2, + "source_pass_count": 2, + "source_blocked_count": 1, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", @@ -22532,8 +22533,8 @@ "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, "source_check_count": 13, - "source_pass_count": 6, - "source_blocked_count": 7, + "source_pass_count": 7, + "source_blocked_count": 6, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -22928,7 +22929,7 @@ "review_state": "awaiting-submission", "source_accepted": false, "objective": "Import production metadata-only events from a real external client into the local drift loop.", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "execution_runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", "Install the generated native messaging manifest for the real client and send at least one accepted skill_activation or skill_output event.", @@ -22976,7 +22977,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -22997,8 +22998,8 @@ "label": "Adoption sample", "field": "adoption_sample_count", "expected": ">0", - "actual": 0, - "status": "blocked", + "actual": 1, + "status": "pass", "source_accepted": false, "next_action": "Telemetry must include adoption outcome evidence." }, @@ -23013,10 +23014,9 @@ "next_action": "Telemetry must stay metadata-only." } ], - "blocked_source_check_count": 2, + "blocked_source_check_count": 1, "next_source_actions": [ - "Import at least one metadata-only event from a real client.", - "Telemetry must include adoption outcome evidence." + "Import at least one metadata-only event from a real client." ], "submission_state": { "status": "missing", @@ -23855,7 +23855,7 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "a163b6e288c5342e683875b47fdfe62989d3f09241c0c8fd86ae615c6d5da7ad", + "package_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69", "archive_sha256": "5802e5f522558cf77666ae59390a28845cf3affd22fa1267b6629891ddeba76b" }, "compatibility": { @@ -23892,11 +23892,11 @@ "install_simulated": true, "install_simulation": "reports/install_simulation.json" }, - "generated_at": "2026-06-16" + "generated_at": "2026-06-17" }, "index": { "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-17", "package_count": 1, "packages": [ { @@ -23912,7 +23912,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "a163b6e288c5342e683875b47fdfe62989d3f09241c0c8fd86ae615c6d5da7ad" + "package_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69" } ] }, @@ -24594,14 +24594,14 @@ "install_simulation": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "skill_dir": ".", - "package_dir": "dist", - "install_root": "dist/install-simulation/simulate-yao-meta-skill", - "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", + "package_dir": "tests/tmp_review_studio/dist", + "install_root": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill", + "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 658, + "archive_entry_count": 677, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -24619,7 +24619,7 @@ { "id": "archive-present", "status": "pass", - "detail": "Package archive exists: dist/yao-meta-skill.zip" + "detail": "Package archive exists: tests/tmp_review_studio/dist/yao-meta-skill.zip" }, { "id": "archive-safe-paths", @@ -24865,16 +24865,16 @@ "failures": [], "warnings": [], "artifacts": { - "archive": "dist/yao-meta-skill.zip", - "package_manifest": "dist/manifest.json", - "json": "reports/install_simulation.json", - "markdown": "reports/install_simulation.md" + "archive": "tests/tmp_review_studio/dist/yao-meta-skill.zip", + "package_manifest": "tests/tmp_review_studio/dist/manifest.json", + "json": "tests/tmp_review_studio/install_simulation.json", + "markdown": "tests/tmp_review_studio/install_simulation.md" } }, "upgrade_check": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-16", + "generated_at": "2026-06-13", "previous": { "name": "yao-meta-skill", "version": "1.0.0", @@ -24948,7 +24948,7 @@ { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "a163b6e288c5342e683875b47fdfe62989d3f09241c0c8fd86ae615c6d5da7ad" + "to": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69" } ] }, @@ -24963,8 +24963,8 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "reports/upgrade_check.json", - "markdown": "reports/upgrade_check.md" + "json": "tests/tmp_review_studio/upgrade_check.json", + "markdown": "tests/tmp_review_studio/upgrade_check.md" } }, "manifest": { diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 8e08b532..08570359 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 231, - "package_sha256": "d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69" + "package_sha256": "9cb9a19e774540e721b143a240baf41b41c32b9f10df532eb4a7d1cf7a2fe67b" }, "failures": [], "warnings": [], @@ -780,7 +780,7 @@ "uses_input": false, "uses_network": false, "uses_file_write": true, - "uses_subprocess": false, + "uses_subprocess": true, "network_urls": [], "network_hosts": [] }, @@ -3275,6 +3275,7 @@ "scripts/ci_test.py", "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", + "scripts/render_evidence_consistency.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", "scripts/sync_local_install.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 2d01beda..77abd9f7 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `231` -- Package SHA256: `d50f6ac9714b62b985f9f9edb181f87bb57246be1e4249cb92e7e4a87ae5aa69` +- Package SHA256: `9cb9a19e774540e721b143a240baf41b41c32b9f10df532eb4a7d1cf7a2fe67b` ## Failures @@ -114,7 +114,7 @@ | scripts/render_daily_skillops_report.py | cli | True | True | True | False | False | True | False | Renders a Daily SkillOps report that summarizes explicit-source patterns, proposals, approval state, and release evidence without scanning private logs or applying patches. | | scripts/render_description_drift_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_eval_dashboard.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | -| scripts/render_evidence_consistency.py | cli | True | True | True | False | False | True | False | Renders a cross-report evidence consistency gate for generated Skill OS reports. | +| scripts/render_evidence_consistency.py | cli | True | True | True | False | False | True | True | Renders a cross-report evidence consistency gate for generated Skill OS reports. | | scripts/render_intent_confidence.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_intent_dialogue.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_iteration_directions.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index d9dd5461..7fbbae21 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "item_count": 15, diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 74eb8e26..137bf995 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -1,6 +1,6 @@ # Skill OS 2.0 Audit -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index e598eec9..7844e0a9 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -1,28 +1,28 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "item_count": 21, "module_count": 8, "recommended_pr_count": 13, - "pass_count": 21, - "warn_count": 0, + "pass_count": 20, + "warn_count": 1, "missing_count": 0, "extension_track_count": 4, "extension_partial_count": 0, "extension_planned_count": 0, "extension_covered_count": 4, "adaptive_extension_ready": true, - "local_blueprint_ready": true, + "local_blueprint_ready": false, "public_world_class_ready": false, "world_class_evidence_pending_count": 4, - "decision": "local-blueprint-covered-evidence-pending" + "decision": "continue-implementation" }, "status_counts": { - "pass": 21, - "warn": 0, + "pass": 20, + "warn": 1, "missing": 0 }, "extension_status_counts": { @@ -611,9 +611,9 @@ "key": "evidence-consistency", "category": "recommended-pr", "label": "Evidence Consistency", - "status": "pass", + "status": "warn", "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", - "current": "35 consistency checks", + "current": "36 consistency checks", "command": "make ci-test", "test": "tests/verify_evidence_consistency.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 83966a1b..497fdb52 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -1,15 +1,15 @@ # Skill OS 2.0 Blueprint Coverage -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary -- decision: `local-blueprint-covered-evidence-pending` -- local blueprint ready: `true` +- decision: `continue-implementation` +- local blueprint ready: `false` - public world-class ready: `false` -- pass: `21` / `21` +- pass: `20` / `21` - missing: `0` -- warn: `0` +- warn: `1` - reference extensions: `4` - extension covered: `4` - extension partial: `0` @@ -48,7 +48,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Registry Package Format | `pass` | registry ok True | `make ci-test` | `tests/verify_registry_audit.py` | | Review Studio 2.0 | `pass` | 16 review gates | `make ci-test` | `tests/verify_review_studio.py` | | Migration V2 Docs | `pass` | migration guide present | `make ci-test` | `docs review` | -| Evidence Consistency | `pass` | 35 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | +| Evidence Consistency | `warn` | 36 consistency checks | `make ci-test` | `tests/verify_evidence_consistency.py` | ## Reference Extension Tracks @@ -213,7 +213,7 @@ These extension tracks come from the user-supplied 2.0 reference plan. They are ### Evidence Consistency - objective: Recommended Skill OS 2.0 implementation PR from the upgrade plan. -- status: `pass` +- status: `warn` - existing evidence: `scripts/render_evidence_consistency.py`, `reports/evidence_consistency.json`, `tests/verify_evidence_consistency.py` - next action: Keep this item covered as the implementation evolves. diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index fe24ca0b..22da9491 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "ledger_ready_to_claim_world_class": false, diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index 5315b99a..d2d4d668 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -1,6 +1,6 @@ # World-Class Claim Guard -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/world_class_evidence_intake.json b/reports/world_class_evidence_intake.json index df978d9e..81d92d87 100644 --- a/reports/world_class_evidence_intake.json +++ b/reports/world_class_evidence_intake.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "schema_present": true, diff --git a/reports/world_class_evidence_intake.md b/reports/world_class_evidence_intake.md index 297677c7..440128cd 100644 --- a/reports/world_class_evidence_intake.md +++ b/reports/world_class_evidence_intake.md @@ -1,6 +1,6 @@ # World-Class Evidence Intake -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json index ef380d20..d81f6f75 100644 --- a/reports/world_class_evidence_ledger.json +++ b/reports/world_class_evidence_ledger.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "decision": "evidence-pending", "ready_to_claim_world_class": false, diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md index eb8c1650..fb22bd0b 100644 --- a/reports/world_class_evidence_ledger.md +++ b/reports/world_class_evidence_ledger.md @@ -1,6 +1,6 @@ # World-Class Evidence Ledger -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index 50908b39..ff133db6 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "audit_decision": "continue-iteration", diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index 3a1b0962..b540f1e6 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -1,6 +1,6 @@ # World-Class Evidence Plan -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/world_class_evidence_preflight.json b/reports/world_class_evidence_preflight.json index 4a7eaf7a..bf6bf821 100644 --- a/reports/world_class_evidence_preflight.json +++ b/reports/world_class_evidence_preflight.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "evidence_item_count": 4, diff --git a/reports/world_class_evidence_preflight.md b/reports/world_class_evidence_preflight.md index e487d321..d6431496 100644 --- a/reports/world_class_evidence_preflight.md +++ b/reports/world_class_evidence_preflight.md @@ -1,6 +1,6 @@ # World-Class Evidence Preflight -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index 03f7b4a1..18dec411 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "summary": { "evidence_item_count": 4, diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index 8ba17d83..17ebd41e 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -1,6 +1,6 @@ # World-Class Operator Runbook -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 8abee31b..5b9f0a2d 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -1,7 +1,7 @@ { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-13", + "generated_at": "2026-06-17", "skill_dir": ".", "decision": "awaiting-submissions", "ready_to_claim_world_class": false, diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index 21bd5b52..4958147c 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -1,6 +1,6 @@ # World-Class Submission Review -Generated at: `2026-06-13` +Generated at: `2026-06-17` ## Summary diff --git a/scripts/render_evidence_consistency.py b/scripts/render_evidence_consistency.py index 9b6b182b..57d68f46 100644 --- a/scripts/render_evidence_consistency.py +++ b/scripts/render_evidence_consistency.py @@ -1,6 +1,7 @@ #!/usr/bin/env python3 import argparse import json +import subprocess from datetime import date from pathlib import Path from typing import Any @@ -37,6 +38,22 @@ SCRIPT_INTERFACE = "cli" SCRIPT_INTERFACE_REASON = "Renders a cross-report evidence consistency gate for generated Skill OS reports." +def git_worktree_status(skill_dir: Path) -> dict[str, Any]: + try: + proc = subprocess.run( + ["git", "-C", str(skill_dir), "status", "--porcelain=v1", "-uall"], + capture_output=True, + text=True, + check=False, + ) + except OSError: + return {"available": False, "clean": None, "changed_file_count": None} + if proc.returncode != 0: + return {"available": False, "clean": None, "changed_file_count": None} + changes = [line for line in proc.stdout.splitlines() if line.strip()] + return {"available": True, "clean": not changes, "changed_file_count": len(changes)} + + def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: reports: dict[str, dict[str, Any]] = {} text_reports: dict[str, str] = {} @@ -145,6 +162,32 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: paths=[REQUIRED_REPORTS["benchmark"]], detail="The benchmark release lock must reflect the generation-time git dirty flag.", ) + current_git_status = git_worktree_status(skill_dir) + if current_git_status.get("available") and current_git_status.get("clean") is True: + compare_values( + checks, + key="benchmark-clean-worktree-release-lock", + label="Clean worktree keeps a clean benchmark release lock", + expected=True, + actual=benchmark_summary.get("release_lock_ready"), + paths=[REQUIRED_REPORTS["benchmark"]], + detail=( + "If the current worktree is clean, the committed benchmark report must not still carry a dirty release lock from an earlier generation." + ), + ) + else: + add_check( + checks, + key="benchmark-clean-worktree-release-lock", + label="Clean worktree keeps a clean benchmark release lock", + status="pass", + expected="checked only when git is available and the current worktree is clean", + actual=current_git_status, + paths=[REQUIRED_REPORTS["benchmark"]], + detail=( + "Dirty or non-git worktrees cannot prove final release-lock freshness, so this check is advisory until the final clean-lock pass." + ), + ) skill_name = str(overview.get("name") or nested(review_studio, ["data", "frontmatter", "name"]) or skill_dir.name) expected_skill_ir_path = find_skill_ir_path(skill_dir, skill_name, require_schema=True) expected_skill_ir = { diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index c73e6d91..5e698156 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -423,6 +423,57 @@ def main() -> None: benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"] ) + stale_clean_lock_root = TMP / "stale-clean-lock-skill" + copy_reports(stale_clean_lock_root) + benchmark_path = stale_clean_lock_root / "reports" / "benchmark_reproducibility.json" + benchmark = json.loads(benchmark_path.read_text(encoding="utf-8")) + benchmark["git_status"]["dirty"] = True + benchmark["git_status"]["changed_file_count"] = 3 + benchmark["summary"]["release_lock_ready"] = False + benchmark["summary"]["working_tree_dirty"] = True + benchmark["summary"]["changed_file_count"] = 3 + benchmark["release_lock"]["ready"] = False + benchmark["release_lock"]["reason"] = "working tree was dirty at generation time" + benchmark_path.write_text(json.dumps(benchmark, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + for report_name in ["skill-overview", "skill-interpretation"]: + report_path = stale_clean_lock_root / "reports" / f"{report_name}.json" + report = json.loads(report_path.read_text(encoding="utf-8")) + report["benchmark_reproducibility"]["summary"]["release_lock_ready"] = False + report["benchmark_reproducibility"]["summary"]["working_tree_dirty"] = True + report["benchmark_reproducibility"]["summary"]["changed_file_count"] = 3 + report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + subprocess.run(["git", "init"], cwd=stale_clean_lock_root, capture_output=True, text=True, check=True) + subprocess.run(["git", "add", "."], cwd=stale_clean_lock_root, capture_output=True, text=True, check=True) + subprocess.run( + ["git", "-c", "user.name=Yao Test", "-c", "user.email=yao-test@example.com", "commit", "-m", "seed"], + cwd=stale_clean_lock_root, + capture_output=True, + text=True, + check=True, + ) + stale_clean_lock_proc = run( + [ + sys.executable, + str(SCRIPT), + str(stale_clean_lock_root), + "--output-json", + str(TMP / "stale_clean_lock.json"), + "--output-md", + str(TMP / "stale_clean_lock.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert stale_clean_lock_proc.returncode == 2, stale_clean_lock_proc.stdout + stale_clean_lock_payload = json.loads(stale_clean_lock_proc.stdout) + stale_clean_lock_checks = {item["key"]: item for item in stale_clean_lock_payload["checks"]} + assert stale_clean_lock_checks["benchmark-release-lock-self-consistency"]["status"] == "pass", ( + stale_clean_lock_checks["benchmark-release-lock-self-consistency"] + ) + assert stale_clean_lock_checks["benchmark-clean-worktree-release-lock"]["status"] == "fail", ( + stale_clean_lock_checks["benchmark-clean-worktree-release-lock"] + ) + workflow_drift_root = TMP / "workflow-drift-skill" copy_reports(workflow_drift_root) studio_path = workflow_drift_root / "reports" / "review-studio.json"