diff --git a/registry/index.json b/registry/index.json index 31b6c8a..4918bdb 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index feda5cc..c9722f0 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5", - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 9932c69..5b66b9c 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 576, + "archive_entry_count": 577, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 0ec03dd..3702a83 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.5, + "duration_ms": 26.18, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.91, + "duration_ms": 26.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.88, + "duration_ms": 27.99, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.4, + "duration_ms": 29.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.19, + "duration_ms": 29.88, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.57, + "duration_ms": 27.32, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.78, + "duration_ms": 26.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.9, + "duration_ms": 26.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.33, + "duration_ms": 25.85, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.23, + "duration_ms": 26.14, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 9231949..ec37f1c 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 30.5 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 30.91 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 32.88 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 33.4 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 30.19 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 30.57 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 30.78 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 29.9 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 30.33 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 33.23 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.18 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.25 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 27.99 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 29.37 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 29.88 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 27.32 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.55 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.19 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.85 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.14 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 6a9c01e..b0d9ac5 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", - "archive_entry_count": 576, + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", + "archive_entry_count": 577, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 67c12d9..721616a 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc` +- Archive SHA256: `36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 19b6380..07417d0 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5", - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 428be2e..93387f0 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5` -- Archive SHA256: `11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc` +- Package SHA256: `52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a` +- Archive SHA256: `36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 813c53b..893b65a 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -220,6 +220,44 @@ margin: 0; padding-left: 18px; } + .world-source-panel { + border-top: 1px solid var(--line); + padding-top: 12px; + } + .world-source-panel h4 { + margin: 0 0 8px; + color: var(--ink); + font-size: 14px; + font-weight: 500; + } + .world-source-checks { + list-style: none; + padding: 0; + margin: 0; + display: grid; + gap: 8px; + } + .world-source-check { + display: grid; + grid-template-columns: minmax(0, .7fr) minmax(0, .6fr) minmax(0, 1fr); + gap: 10px; + align-items: start; + min-width: 0; + padding: 9px 0 0; + border-top: 1px solid var(--line); + } + .world-source-check span { + color: var(--ink); + font-size: 14px; + } + .world-source-check code, + .world-source-check small { + color: var(--muted); + font-size: 13px; + overflow-wrap: anywhere; + } + .world-source-check.pass { border-left: 3px solid var(--pass); padding-left: 8px; } + .world-source-check.blocked { border-left: 3px solid var(--warn); padding-left: 8px; } .output-review-grid { display: grid; grid-template-columns: minmax(0, 1fr); @@ -482,7 +520,7 @@ font-size: 13px; } @media (max-width: 980px) { - .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .output-review-grid, .output-review-steps, .world-evidence-grid, .world-evidence-columns, .world-intake-grid, .world-intake-steps, .waiver-candidate-grid, .waiver-card dl, .kv-grid { grid-template-columns: 1fr; } + .metrics, .gates, .twocol, .actions-grid, .annotations-grid, .output-review-grid, .output-review-steps, .world-evidence-grid, .world-evidence-columns, .world-source-check, .world-intake-grid, .world-intake-steps, .waiver-candidate-grid, .waiver-card dl, .kv-grid { grid-template-columns: 1fr; } main { padding: 32px 18px 60px; } nav { justify-content: flex-start; overflow-x: auto; flex-wrap: nowrap; } nav a { flex: 0 0 auto; } @@ -505,12 +543,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

3 blockers; local reproducible true

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

97 scripts scanned; secrets found

Py Compat0

159 files scanned for Python 3.11

Arch Debt0

891 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

73 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

576 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Public Claimblocked

3 blockers; local reproducible true

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

98 scripts scanned; secrets found

Py Compat0

159 files scanned for Python 3.11

Arch Debt0

891 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

73 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

577 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 960/1000; deferred 388370/120000; top deferred scripts 340634; resource governance governed; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 97 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 159 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

156 Python files; 0 hotspots; 0 blockers; largest 891 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 960/1000; deferred 388370/120000; top deferred scripts 340634; resource governance governed; quality density 135.4

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 98 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 159 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

156 Python files; 0 hotspots; 0 blockers; largest 891 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -577,7 +615,7 @@
-

信任报告

Secret
0
脚本数
97
网络脚本
3
Help 失败
0
包体哈希
25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5
+

信任报告

Secret
0
脚本数
98
网络脚本
3
Help 失败
0
包体哈希
52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -620,7 +658,7 @@

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.

源证据检查

  • Provider model runmodel_executed_count: 0 / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: 0 / >0Provider execution should return non-estimated token usage.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.

源证据检查

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: 0 / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: 0 / ==0Fix malformed winner/confidence entries.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.

源证据检查

  • Native enforcementnative_enforcement_count: 0 / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: 0 / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.

源证据检查

  • External eventsexternal_source_events: 0 / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: 0 / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: False / falseTelemetry must stay metadata-only.
@@ -656,12 +694,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
576
失败数
0
警告数
0
归档哈希
11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
577
失败数
0
警告数
0
归档哈希
36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e
diff --git a/reports/review-studio.json b/reports/review-studio.json index 0ffdd05..8ced0b9 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 97 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 98 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -851,7 +851,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 97 + "file_count": 98 }, { "path": "evals", @@ -866,7 +866,7 @@ "file_count": 203 } ], - "file_count": 364, + "file_count": 365, "folder_count": 4, "distribution": [ { @@ -891,7 +891,7 @@ }, { "label": "scripts", - "value": 97 + "value": 98 }, { "label": "evals", @@ -1020,7 +1020,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 97 + "file_count": 98 }, { "path": "evals", @@ -1441,9 +1441,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 181, - "script_count": 97, - "internal_module_count": 20, + "scanned_files": 182, + "script_count": 98, + "internal_module_count": 21, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1461,8 +1461,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 181, - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_hash_file_count": 182, + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" }, "skill_atlas": { "skill_count": 12, @@ -1500,8 +1500,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5", - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, "compatibility": { "openai": "pass", @@ -1532,7 +1532,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1548,8 +1548,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", - "archive_entry_count": 576, + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", + "archive_entry_count": 577, "failure_count": 0, "warning_count": 0 }, @@ -1560,7 +1560,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 576, + "archive_entry_count": 577, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1627,12 +1627,12 @@ { "field": "archive_sha256", "from": "", - "to": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "to": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "to": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] }, @@ -3811,7 +3811,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.5, + "duration_ms": 26.18, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3839,7 +3839,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.91, + "duration_ms": 26.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3862,7 +3862,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.88, + "duration_ms": 27.99, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3890,7 +3890,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.4, + "duration_ms": 29.37, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3913,7 +3913,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.19, + "duration_ms": 29.88, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3941,7 +3941,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.57, + "duration_ms": 27.32, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3964,7 +3964,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.78, + "duration_ms": 26.55, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3991,7 +3991,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.9, + "duration_ms": 26.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4014,7 +4014,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.33, + "duration_ms": 25.85, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4043,7 +4043,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.23, + "duration_ms": 26.14, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -5272,7 +5272,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "97 scripts; secrets 0; help failures 0", + "current": "98 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -5346,7 +5346,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 576; install failures 0", + "current": "archive entries 577; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ @@ -10632,9 +10632,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 181, - "script_count": 97, - "internal_module_count": 20, + "scanned_files": 182, + "script_count": 98, + "internal_module_count": 21, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -10652,8 +10652,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 181, - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_hash_file_count": 182, + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" }, "failures": [], "warnings": [], @@ -11936,6 +11936,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_source_checks.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to keep source-evidence readiness checks consistent.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao.py", "interface": "cli", @@ -12069,7 +12083,7 @@ "checked_count": 77, "passed_count": 77, "failed_count": 0, - "skipped_count": 20, + "skipped_count": 21, "failed_scripts": [], "results": [ { @@ -12900,6 +12914,10 @@ "path": "scripts/world_class_evidence_contract.py", "reason": "internal module" }, + { + "path": "scripts/world_class_source_checks.py", + "reason": "internal module" + }, { "path": "scripts/yao_cli_config.py", "reason": "internal module" @@ -16935,6 +16953,9 @@ "fix_submission_count": 0, "unmatched_submission_count": 0, "invalid_submission_count": 0, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -16966,6 +16987,41 @@ "token_observed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/output_execution_runs.json summary.model_executed_count > 0", "reports/output_execution_runs.json summary.timing_observed_count > 0", @@ -17002,6 +17058,51 @@ "answer_revealed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + } + ], + "source_check_count": 4, + "source_pass_count": 2, + "source_blocked_count": 2, "success_checks": [ "reports/output_review_adjudication.json summary.pending_count == 0", "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", @@ -17040,6 +17141,41 @@ "failure_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", "reports/runtime_permission_probes.json summary.failure_count == 0", @@ -17075,6 +17211,41 @@ "risk_band": "low", "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", @@ -17110,6 +17281,9 @@ "ready_for_ledger_review_count": 0, "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -17172,6 +17346,38 @@ "token_observed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", @@ -17244,6 +17450,48 @@ "answer_revealed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/human-adjudication.json", @@ -17319,6 +17567,38 @@ "failure_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-permission-enforcement.json", @@ -17388,6 +17668,38 @@ "risk_band": "low", "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", @@ -17783,8 +18095,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5", - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, "compatibility": { "openai": "pass", @@ -17815,7 +18127,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -17840,7 +18152,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] }, @@ -17863,8 +18175,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", - "archive_entry_count": 576, + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", + "archive_entry_count": 577, "failure_count": 0, "warning_count": 0 }, @@ -18529,7 +18841,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 576, + "archive_entry_count": 577, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -18871,12 +19183,12 @@ { "field": "archive_sha256", "from": "", - "to": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "to": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "to": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] }, diff --git a/reports/review-viewer.json b/reports/review-viewer.json index 6290310..6ffeb49 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -438,7 +438,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 97 + "file_count": 98 }, { "path": "evals", @@ -453,7 +453,7 @@ "file_count": 203 } ], - "file_count": 364, + "file_count": 365, "folder_count": 4, "distribution": [ { @@ -478,7 +478,7 @@ }, { "label": "scripts", - "value": 97 + "value": 98 }, { "label": "evals", @@ -607,7 +607,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 97 + "file_count": 98 }, { "path": "evals", @@ -1028,9 +1028,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 181, - "script_count": 97, - "internal_module_count": 20, + "scanned_files": 182, + "script_count": 98, + "internal_module_count": 21, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1048,8 +1048,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 181, - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_hash_file_count": 182, + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" }, "skill_atlas": { "skill_count": 12, @@ -1087,8 +1087,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5", - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, "compatibility": { "openai": "pass", @@ -1119,7 +1119,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1135,8 +1135,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", - "archive_entry_count": 576, + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", + "archive_entry_count": 577, "failure_count": 0, "warning_count": 0 }, @@ -1147,7 +1147,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 576, + "archive_entry_count": 577, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1214,12 +1214,12 @@ { "field": "archive_sha256", "from": "", - "to": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "to": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "to": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 58a7b40..cae6e1e 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 181, - "script_count": 97, - "internal_module_count": 20, + "scanned_files": 182, + "script_count": 98, + "internal_module_count": 21, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 181, - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_hash_file_count": 182, + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" }, "failures": [], "warnings": [], @@ -1306,6 +1306,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_source_checks.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world-class evidence reports to keep source-evidence readiness checks consistent.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/yao.py", "interface": "cli", @@ -1439,7 +1453,7 @@ "checked_count": 77, "passed_count": 77, "failed_count": 0, - "skipped_count": 20, + "skipped_count": 21, "failed_scripts": [], "results": [ { @@ -2270,6 +2284,10 @@ "path": "scripts/world_class_evidence_contract.py", "reason": "internal module" }, + { + "path": "scripts/world_class_source_checks.py", + "reason": "internal module" + }, { "path": "scripts/yao_cli_config.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index aae7455..751c2d8 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `181` -- Scripts: `97` -- Internal script modules: `20` +- Scanned files: `182` +- Scripts: `98` +- Internal script modules: `21` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `181` -- Package SHA256: `25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5` +- Package hash files: `182` +- Package SHA256: `52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a` ## Failures @@ -152,6 +152,7 @@ | scripts/validate_skill.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | +| scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. | | scripts/yao.py | cli | False | True | True | False | False | False | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/yao_cli_config.py | internal-module | True | False | False | False | False | False | False | Imported by yao.py for CLI target maps and side-effect-free shaping helpers. | | scripts/yao_cli_create_commands.py | internal-module | True | True | False | False | False | False | False | Imported by yao.py to keep skill creation and quickstart command handlers out of the CLI orchestrator. | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index d2ed34d..275eba7 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布364项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布365项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index c27e657..84ac870 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -437,7 +437,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 97 + "file_count": 98 }, { "path": "evals", @@ -452,7 +452,7 @@ "file_count": 203 } ], - "file_count": 364, + "file_count": 365, "folder_count": 4, "distribution": [ { @@ -477,7 +477,7 @@ }, { "label": "scripts", - "value": 97 + "value": 98 }, { "label": "evals", @@ -606,7 +606,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 97 + "file_count": 98 }, { "path": "evals", @@ -1027,9 +1027,9 @@ "failures": [] }, "trust_security": { - "scanned_files": 181, - "script_count": 97, - "internal_module_count": 20, + "scanned_files": 182, + "script_count": 98, + "internal_module_count": 21, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -1047,8 +1047,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 181, - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "package_hash_file_count": 182, + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" }, "skill_atlas": { "skill_count": 12, @@ -1086,8 +1086,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5", - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "package_sha256": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, "compatibility": { "openai": "pass", @@ -1118,7 +1118,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1134,8 +1134,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc", - "archive_entry_count": 576, + "archive_sha256": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e", + "archive_entry_count": 577, "failure_count": 0, "warning_count": 0 }, @@ -1146,7 +1146,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 576, + "archive_entry_count": 577, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1213,12 +1213,12 @@ { "field": "archive_sha256", "from": "", - "to": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "to": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "to": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] }, diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 6588bfb..eec2f10 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -122,7 +122,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "97 scripts; secrets 0; help failures 0", + "current": "98 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ @@ -196,7 +196,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 576; install failures 0", + "current": "archive entries 577; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 8c33833..3a60763 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -21,9 +21,9 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 97 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 98 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 576; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 577; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 4519f56..2f56719 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "11c75419dc6599e7c555a153e4d72127a2c32a73d9426ecde4d110e4777453dc" + "to": "36ec77ec0badfe8b44286434d0c10de0470dee241f33b569ea24a0e971721a0e" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "25098dc5932e847a1aeb06f51ada0de60fec1382fe286fc88e9bc5e60fb4d3c5" + "to": "52472a5f16d07e88959326df916a03a327b3f3c9443d2128556002c431c9f70a" } ] }, diff --git a/reports/world_class_operator_runbook.html b/reports/world_class_operator_runbook.html index 665bcc1..2d4b687 100644 --- a/reports/world_class_operator_runbook.html +++ b/reports/world_class_operator_runbook.html @@ -22,7 +22,7 @@ h3 { margin:4px 0 10px; font-size:22px; } h4 { margin:0 0 8px; font-size:16px; } .lede { max-width:800px; color:var(--muted); font-size:20px; } - .stats { display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin-top:26px; } + .stats { display:grid; grid-template-columns:repeat(5, minmax(0,1fr)); gap:12px; margin-top:26px; } .stats article, .panel, .item-card { border:1px solid var(--line); border-radius:8px; background:#fff; } .stats article { padding:16px; } .stats span, .item-card span, .muted { color:var(--muted); } @@ -42,6 +42,11 @@ .mini-grid { display:grid; grid-template-columns:repeat(3, minmax(0,1fr)); gap:12px; margin-top:14px; } .mini-grid section, .panel { background:var(--soft); border-radius:8px; padding:14px; min-width:0; } .mini-grid li, .panel li { overflow-wrap:anywhere; } + .source-panel { background:var(--soft); border-radius:8px; padding:14px; min-width:0; } + .source-checks { list-style:none; padding:0; margin:0; display:grid; gap:8px; } + .source-check { border-top:1px solid var(--line); padding-top:8px; display:grid; gap:3px; } + .source-check span { color:var(--ink); } + .source-check code, .source-check small { overflow-wrap:anywhere; } @media (max-width:820px) { .stats, .mini-grid { grid-template-columns:1fr; } h1 { font-size:38px; } .topbar-inner { align-items:flex-start; flex-direction:column; } } @@ -52,7 +57,7 @@ Evidence Operations

World-Class Operator Runbook

A single operating page for collecting the remaining human and external evidence. It coordinates action, but does not accept evidence or change the ledger.

-
Pending4
Awaiting4
Ready
Invalid
+
Pending4
Awaiting4
Ready
Source6/13
Invalid

Fast Path

  1. Run the real external or human work for one evidence item.
  2. Generate and fill the matching submission draft.
  3. Validate intake and inspect the submission review queue.
  4. Refresh the ledger and run the claim guard before making any completion claim.

Evidence Items

@@ -69,6 +74,7 @@

Success Checks

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

Privacy

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
+

Source Evidence Snapshot

  • Provider model runmodel_executed_count: / >0Run provider-backed output-exec with real credentials.
  • Timing observedtiming_observed_count: 10 / >0Provider execution should record timing metadata.
  • Token usage observedtoken_observed_count: / >0Provider execution should return non-estimated token usage.
human · awaiting-submission

Human Adjudication

No evidence packet has been submitted for review.

@@ -83,6 +89,7 @@

Success Checks

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

Privacy

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
+

Source Evidence Snapshot

  • Review pairs existpair_count: 5 / >0Generate the blind A/B review pack.
  • No pending decisionspending_count: 5 / ==0Record a reviewer choice for every pair.
  • Judgments completejudgment_count: / ==pair_countEvery pair needs one valid human judgment.
  • No invalid decisionsinvalid_decision_count: / ==0Fix malformed winner/confidence entries.
external · awaiting-submission

Native Permission Enforcement

No evidence packet has been submitted for review.

@@ -97,6 +104,7 @@

Success Checks

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

Privacy

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
+

Source Evidence Snapshot

  • Native enforcementnative_enforcement_count: / >0Collect real target-client or external runtime guard proof.
  • Probe failuresfailure_count: / ==0Runtime permission probes must stay clean.
  • Installer supportinstaller_enforcement_ready: True / trueInstaller enforcement is supporting evidence, not native proof.
external · awaiting-submission

Native Client Telemetry

No evidence packet has been submitted for review.

@@ -111,6 +119,7 @@

Success Checks

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

Privacy

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+

Source Evidence Snapshot

  • External eventsexternal_source_events: / >0Import at least one metadata-only event from a real client.
  • Adoption sampleadoption_sample_count: / >0Telemetry must include adoption outcome evidence.
  • Raw content blockedraw_content_allowed: / falseTelemetry must stay metadata-only.

Boundary

diff --git a/reports/world_class_operator_runbook.json b/reports/world_class_operator_runbook.json index 81d2dfa..20e868a 100644 --- a/reports/world_class_operator_runbook.json +++ b/reports/world_class_operator_runbook.json @@ -11,6 +11,9 @@ "ready_for_ledger_review_count": 0, "valid_packet_source_incomplete_count": 0, "invalid_submission_count": 0, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, "ready_to_claim_world_class": false, "runbook_counts_as_completion": false, "decision": "collect-evidence" @@ -73,6 +76,38 @@ "token_observed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/provider-holdout.json", @@ -145,6 +180,48 @@ "answer_revealed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/human-adjudication.json", @@ -220,6 +297,38 @@ "failure_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-permission-enforcement.json", @@ -289,6 +398,38 @@ "risk_band": "low", "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], "submission_state": { "status": "missing", "path": "evidence/world_class/submissions/native-client-telemetry.json", diff --git a/reports/world_class_operator_runbook.md b/reports/world_class_operator_runbook.md index bdcc1f0..4bf0af7 100644 --- a/reports/world_class_operator_runbook.md +++ b/reports/world_class_operator_runbook.md @@ -73,6 +73,14 @@ This runbook coordinates evidence collection only. It does not accept submission - reports/world_class_evidence_intake.json - reports/world_class_evidence_intake.md +### Source Evidence Snapshot + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| Provider model run | `0` | `>0` | `blocked` | +| Timing observed | `10` | `>0` | `pass` | +| Token usage observed | `0` | `>0` | `blocked` | + ## Human Adjudication - objective: Record real blind A/B reviewer decisions before claiming human output review completion. @@ -118,6 +126,15 @@ This runbook coordinates evidence collection only. It does not accept submission - reports/world_class_evidence_intake.json - reports/world_class_evidence_intake.md +### Source Evidence Snapshot + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| Review pairs exist | `5` | `>0` | `pass` | +| No pending decisions | `5` | `==0` | `blocked` | +| Judgments complete | `0` | `==pair_count` | `blocked` | +| No invalid decisions | `0` | `==0` | `pass` | + ## Native Permission Enforcement - objective: Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities. @@ -164,6 +181,14 @@ This runbook coordinates evidence collection only. It does not accept submission - reports/world_class_evidence_intake.json - reports/world_class_evidence_intake.md +### Source Evidence Snapshot + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| Native enforcement | `0` | `>0` | `blocked` | +| Probe failures | `0` | `==0` | `pass` | +| Installer support | `True` | `true` | `pass` | + ## Native Client Telemetry - objective: Import production metadata-only events from a real external client into the local drift loop. @@ -207,6 +232,14 @@ This runbook coordinates evidence collection only. It does not accept submission - reports/world_class_evidence_intake.json - reports/world_class_evidence_intake.md +### Source Evidence Snapshot + +| Check | Current | Expected | Status | +| --- | --- | --- | --- | +| External events | `0` | `>0` | `blocked` | +| Adoption sample | `0` | `>0` | `blocked` | +| Raw content blocked | `False` | `false` | `pass` | + ## Boundary - Planned work, draft packets, metadata fallback, pending human decisions, and local command runners do not count as completion. diff --git a/reports/world_class_submission_review.json b/reports/world_class_submission_review.json index 382f084..1f790aa 100644 --- a/reports/world_class_submission_review.json +++ b/reports/world_class_submission_review.json @@ -12,6 +12,9 @@ "fix_submission_count": 0, "unmatched_submission_count": 0, "invalid_submission_count": 0, + "source_check_count": 13, + "source_pass_count": 6, + "source_blocked_count": 7, "ready_to_claim_world_class": false, "review_counts_submission_as_completion": false, "decision": "awaiting-submissions" @@ -43,6 +46,41 @@ "token_observed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "provider-holdout", + "label": "Provider model run", + "field": "model_executed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Run provider-backed output-exec with real credentials." + }, + { + "evidence_key": "provider-holdout", + "label": "Timing observed", + "field": "timing_observed_count", + "expected": ">0", + "actual": 10, + "status": "pass", + "source_accepted": false, + "next_action": "Provider execution should record timing metadata." + }, + { + "evidence_key": "provider-holdout", + "label": "Token usage observed", + "field": "token_observed_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Provider execution should return non-estimated token usage." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/output_execution_runs.json summary.model_executed_count > 0", "reports/output_execution_runs.json summary.timing_observed_count > 0", @@ -79,6 +117,51 @@ "answer_revealed_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "human-adjudication", + "label": "Review pairs exist", + "field": "pair_count", + "expected": ">0", + "actual": 5, + "status": "pass", + "source_accepted": false, + "next_action": "Generate the blind A/B review pack." + }, + { + "evidence_key": "human-adjudication", + "label": "No pending decisions", + "field": "pending_count", + "expected": "==0", + "actual": 5, + "status": "blocked", + "source_accepted": false, + "next_action": "Record a reviewer choice for every pair." + }, + { + "evidence_key": "human-adjudication", + "label": "Judgments complete", + "field": "judgment_count", + "expected": "==pair_count", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Every pair needs one valid human judgment." + }, + { + "evidence_key": "human-adjudication", + "label": "No invalid decisions", + "field": "invalid_decision_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Fix malformed winner/confidence entries." + } + ], + "source_check_count": 4, + "source_pass_count": 2, + "source_blocked_count": 2, "success_checks": [ "reports/output_review_adjudication.json summary.pending_count == 0", "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", @@ -117,6 +200,41 @@ "failure_count": 0, "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-permission-enforcement", + "label": "Native enforcement", + "field": "native_enforcement_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Collect real target-client or external runtime guard proof." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Probe failures", + "field": "failure_count", + "expected": "==0", + "actual": 0, + "status": "pass", + "source_accepted": false, + "next_action": "Runtime permission probes must stay clean." + }, + { + "evidence_key": "native-permission-enforcement", + "label": "Installer support", + "field": "installer_enforcement_ready", + "expected": "true", + "actual": true, + "status": "pass", + "source_accepted": false, + "next_action": "Installer enforcement is supporting evidence, not native proof." + } + ], + "source_check_count": 3, + "source_pass_count": 2, + "source_blocked_count": 1, "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", "reports/runtime_permission_probes.json summary.failure_count == 0", @@ -152,6 +270,41 @@ "risk_band": "low", "accepted": false }, + "source_checklist": [ + { + "evidence_key": "native-client-telemetry", + "label": "External events", + "field": "external_source_events", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Import at least one metadata-only event from a real client." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Adoption sample", + "field": "adoption_sample_count", + "expected": ">0", + "actual": 0, + "status": "blocked", + "source_accepted": false, + "next_action": "Telemetry must include adoption outcome evidence." + }, + { + "evidence_key": "native-client-telemetry", + "label": "Raw content blocked", + "field": "raw_content_allowed", + "expected": "false", + "actual": false, + "status": "pass", + "source_accepted": false, + "next_action": "Telemetry must stay metadata-only." + } + ], + "source_check_count": 3, + "source_pass_count": 1, + "source_blocked_count": 2, "success_checks": [ "reports/adoption_drift_report.json summary.source_types.external > 0", "reports/adoption_drift_report.json summary.adoption_sample_count > 0", diff --git a/reports/world_class_submission_review.md b/reports/world_class_submission_review.md index f8f7527..741ec2c 100644 --- a/reports/world_class_submission_review.md +++ b/reports/world_class_submission_review.md @@ -40,6 +40,12 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks +- Provider model run: 0 / >0 => blocked +- Timing observed: 10 / >0 => pass +- Token usage observed: 0 / >0 => blocked + +#### Completion Assertions + - reports/output_execution_runs.json summary.model_executed_count > 0 - reports/output_execution_runs.json summary.timing_observed_count > 0 - reports/output_execution_runs.json summary.token_observed_count > 0 @@ -66,6 +72,13 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks +- Review pairs exist: 5 / >0 => pass +- No pending decisions: 5 / ==0 => blocked +- Judgments complete: 0 / ==pair_count => blocked +- No invalid decisions: 0 / ==0 => pass + +#### Completion Assertions + - reports/output_review_adjudication.json summary.pending_count == 0 - reports/output_review_adjudication.json summary.judgment_count == summary.pair_count - reports/output_review_adjudication.json summary.invalid_decision_count == 0 @@ -92,6 +105,12 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks +- Native enforcement: 0 / >0 => blocked +- Probe failures: 0 / ==0 => pass +- Installer support: True / true => pass + +#### Completion Assertions + - reports/runtime_permission_probes.json summary.native_enforcement_count > 0 - reports/runtime_permission_probes.json summary.failure_count == 0 - reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence @@ -118,6 +137,12 @@ This report is a read-only reviewer queue. It does not accept evidence or make w #### Source Checks +- External events: 0 / >0 => blocked +- Adoption sample: 0 / >0 => blocked +- Raw content blocked: False / false => pass + +#### Completion Assertions + - reports/adoption_drift_report.json summary.source_types.external > 0 - reports/adoption_drift_report.json summary.adoption_sample_count > 0 - reports/skill_os2_audit.json item native-client-telemetry status becomes pass
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder