From 1e0837122d5c1268dac33fc50f72ca44aa74b07a Mon Sep 17 00:00:00 2001 From: yaojingang Date: Sat, 13 Jun 2026 19:24:33 +0800 Subject: [PATCH] fix: surface pending output review in studio --- references/review-studio-method.md | 2 +- registry/index.json | 2 +- registry/packages/yao-meta-skill.json | 6 +- reports/adoption_drift_report.json | 2 +- reports/context_budget.json | 4 +- reports/output_execution_runs.json | 20 +-- reports/output_execution_runs.md | 20 +-- reports/package_verification.json | 2 +- reports/package_verification.md | 2 +- reports/registry_audit.json | 8 +- reports/registry_audit.md | 4 +- reports/review-studio.html | 18 +-- reports/review-studio.json | 184 ++++++++++++++++++++------ reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/skill-os-2-review.md | 16 +-- reports/skill-overview.html | 8 +- reports/skill-overview.json | 18 +-- reports/upgrade_check.json | 4 +- scripts/render_review_studio.py | 6 +- scripts/review_studio_gates.py | 8 +- tests/verify_review_studio.py | 22 +-- 22 files changed, 234 insertions(+), 126 deletions(-) diff --git a/references/review-studio-method.md b/references/review-studio-method.md index da67a4c..2c52a40 100644 --- a/references/review-studio-method.md +++ b/references/review-studio-method.md @@ -40,7 +40,7 @@ Production, library, and governed reviews should also show a blind A/B review pa When `reports/output_execution_runs.json` exists, Review Studio should show the number of variant runs, command-executed runs, model-executed runs, recorded fixtures, timing-observed runs, and token-estimated runs. Recorded fixtures are valid reproducibility evidence, but they must not be described as model-executed output evidence. -When `reports/output_review_adjudication.json` exists, Review Studio should show reviewed pairs and pending pairs. Pending reviewer decisions are acceptable as an explicit state, but they must not be counted as agreement or human review evidence. Invalid adjudication records should block release because they make the blind review audit untrustworthy. +When `reports/output_review_adjudication.json` exists, Review Studio should show reviewed pairs and pending pairs. Pending reviewer decisions are acceptable as an explicit state, but they must not be counted as agreement or human review evidence. For production, library, and governed packages, pending reviewer decisions should keep the Output Lab in `warn` until reviewer decisions are recorded or the warning is explicitly accepted in the waiver ledger. Invalid adjudication records should block release because they make the blind review audit untrustworthy. The Operations Loop must never display raw telemetry logs. It should link only to `reports/adoption_drift_report.md`; privacy or schema violations are blockers. diff --git a/registry/index.json b/registry/index.json index fef4b96..661ecf0 100644 --- a/registry/index.json +++ b/registry/index.json @@ -15,7 +15,7 @@ "agent-skills-compatible" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 622bec3..d4c2f84 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -15,8 +15,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5", - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, "compatibility": { "openai": "pass", @@ -47,7 +47,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 453be36..d88f91a 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T11:16:31Z", + "generated_at": "2026-06-13T11:23:46Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/context_budget.json b/reports/context_budget.json index 3639880..9b5216c 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,9 +6,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 811, - "other_text_tokens": 871624, + "other_text_tokens": 872858, "estimated_initial_load_tokens": 987, - "estimated_total_text_tokens": 872435, + "estimated_total_text_tokens": 873669, "relevant_file_count": 368, "unused_resource_dirs": [], "quality_signal_points": 130, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 153ce5d..ca2c305 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.51, + "duration_ms": 26.38, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 32.64, + "duration_ms": 27.27, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 38.43, + "duration_ms": 26.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.51, + "duration_ms": 26.45, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.34, + "duration_ms": 26.16, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.94, + "duration_ms": 26.14, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.5, + "duration_ms": 27.24, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.42, + "duration_ms": 28.91, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.18, + "duration_ms": 27.35, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.97, + "duration_ms": 26.7, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 58e706c..2bb6b47 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 32.51 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 32.64 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 38.43 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.51 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 31.34 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 30.94 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 30.5 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 31.42 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 31.18 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 30.97 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.38 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 27.27 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 26.19 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.45 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 26.16 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.14 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 27.24 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.91 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 27.35 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.7 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 56c45c8..e9e6abf 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 3, "adapter_count": 3, "archive_present": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "archive_entry_count": 490, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index caa1f02..a83053c 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `3 / 3` adapters present - Archive present: `True` -- Archive SHA256: `6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451` +- Archive SHA256: `9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 608ace9..69c7d2c 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -20,8 +20,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5", - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, "compatibility": { "openai": "pass", @@ -52,7 +52,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -76,7 +76,7 @@ "agent-skills-compatible" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 8f86738..d670717 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5` -- Archive SHA256: `6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451` +- Package SHA256: `af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068` +- Archive SHA256: `9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 9bfc5df..b3d632a 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -235,8 +235,8 @@

Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.

审查结论 - ready - Score 100/100 + review + Score 92/100
@@ -247,17 +247,17 @@

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
通过

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 987/1000; quality density 131.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 68 scripts; 2 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

3/3 targets probed; native 0; metadata fallback 3; residual risks 3

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
通过

人工批准

0 active waivers cover current warnings

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 987/1000; quality density 131.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 68 scripts; 2 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

3/3 targets probed; native 0; metadata fallback 3; residual risks 3

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据

阻断事项

无。

-

关注事项

无。

+

关注事项

  • 输出实验5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5
  • 人工批准0 active waivers; 1 warning gates still need reviewer decision

修复动作

-

当前没有 blocker 或 warning。保持现有证据链即可。

+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_adjudication.md
验证命令
python3 scripts/run_output_execution.py
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
@@ -313,7 +313,7 @@
-

信任报告

Secret
0
脚本数
68
网络脚本
2
Help 失败
0
包体哈希
d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5
+

信任报告

Secret
0
脚本数
68
网络脚本
2
Help 失败
0
包体哈希
af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -338,18 +338,18 @@
-

人工批准

0 active waivers cover current warnings

+

人工批准

0 active waivers; 1 warning gates still need reviewer decision

批准台账

Waiver Count
0
Active Count
0
Expired Count
0
Invalid Count
0
覆盖 Gate
0

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 3 adapters

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible
兼容通过
6/6
归档哈希
6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible
兼容通过
6/6
归档哈希
9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
3
Adapter
3
归档存在
是
Zip 条目
490
失败数
0
警告数
0
归档哈希
6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451
+

包体验证

目标数
3
Adapter
3
归档存在
是
Zip 条目
490
失败数
0
警告数
0
归档哈希
9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b
diff --git a/reports/review-studio.json b/reports/review-studio.json index 4e6ba69..cb67a6c 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -3,12 +3,12 @@ "ok": true, "skill_dir": ".", "summary": { - "decision": "ready", - "world_class_score": 100, + "decision": "review", + "world_class_score": 92, "gate_count": 13, "blocker_count": 0, - "warning_count": 0, - "action_count": 0, + "warning_count": 2, + "action_count": 2, "annotation_count": 0, "open_annotation_count": 0, "open_annotation_blocker_count": 0, @@ -34,8 +34,8 @@ { "key": "output-lab", "label": "输出实验", - "status": "pass", - "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5", + "status": "warn", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5", "evidence": "reports/output_quality_scorecard.json", "link": "output_quality_scorecard.md" }, @@ -98,8 +98,8 @@ { "key": "review-waivers", "label": "人工批准", - "status": "pass", - "detail": "0 active waivers cover current warnings", + "status": "warn", + "detail": "0 active waivers; 1 warning gates still need reviewer decision", "evidence": "reports/review_waivers.json", "link": "review_waivers.md" }, @@ -121,8 +121,110 @@ } ], "blockers": [], - "warnings": [], - "review_actions": [], + "warnings": [ + { + "key": "output-lab", + "label": "输出实验", + "status": "warn", + "detail": "5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5", + "evidence": "reports/output_quality_scorecard.json", + "link": "output_quality_scorecard.md" + }, + { + "key": "review-waivers", + "label": "人工批准", + "status": "warn", + "detail": "0 active waivers; 1 warning gates still need reviewer decision", + "evidence": "reports/review_waivers.json", + "link": "review_waivers.md" + } + ], + "review_actions": [ + { + "gate_key": "output-lab", + "label": "输出实验", + "status": "warn", + "priority": "warning", + "summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。", + "why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_adjudication.md", + "source_refs": [ + { + "path": "evals/output/cases.jsonl", + "label": "output eval cases", + "kind": "eval", + "line": 1, + "exists": true, + "link": "../evals/output/cases.jsonl" + }, + { + "path": "reports/output_quality_scorecard.md", + "label": "output scorecard", + "kind": "report", + "line": 1, + "exists": true, + "link": "output_quality_scorecard.md" + }, + { + "path": "reports/output_execution_runs.md", + "label": "output execution runs", + "kind": "report", + "line": 1, + "exists": true, + "link": "output_execution_runs.md" + }, + { + "path": "reports/output_blind_review_pack.md", + "label": "blind A/B review pack", + "kind": "report", + "line": 1, + "exists": true, + "link": "output_blind_review_pack.md" + }, + { + "path": "reports/output_review_adjudication.md", + "label": "review adjudication", + "kind": "report", + "line": 1, + "exists": true, + "link": "output_review_adjudication.md" + } + ], + "evidence": "reports/output_quality_scorecard.json", + "evidence_link": "output_quality_scorecard.md", + "verification_command": "python3 scripts/run_output_execution.py" + }, + { + "gate_key": "review-waivers", + "label": "人工批准", + "status": "warn", + "priority": "warning", + "summary": "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。", + "why": "warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。", + "source_fix": "reports/review_waivers.md", + "source_refs": [ + { + "path": "reports/review_waivers.md", + "label": "waiver ledger", + "kind": "report", + "line": 1, + "exists": true, + "link": "review_waivers.md" + }, + { + "path": "references/review-waiver-method.md", + "label": "waiver method", + "kind": "method", + "line": 1, + "exists": true, + "link": "../references/review-waiver-method.md" + } + ], + "evidence": "reports/review_waivers.json", + "evidence_link": "review_waivers.md", + "verification_command": "python3 scripts/render_review_waivers.py ." + } + ], "evidence_paths": { "skill_overview": "reports/skill-overview.html", "review_viewer": "reports/review-viewer.html", @@ -244,7 +346,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 365 个词/字，references 约 14247 个词/字。", + "入口约 365 个词/字，references 约 14278 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -348,7 +450,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。" + "上下文成本需要补强：入口约 365 个词/字，references 约 14278 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -1089,7 +1191,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 152, - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" }, "skill_atlas": { "skill_count": 12, @@ -1123,8 +1225,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5", - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, "compatibility": { "openai": "pass", @@ -1155,7 +1257,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1171,7 +1273,7 @@ "target_count": 3, "adapter_count": 3, "archive_present": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "archive_entry_count": 490, "failure_count": 0, "warning_count": 0 @@ -1245,12 +1347,12 @@ { "field": "archive_sha256", "from": "", - "to": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "to": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "to": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" } ] }, @@ -2978,7 +3080,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.49, + "duration_ms": 26.15, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3006,7 +3108,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.5, + "duration_ms": 25.94, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3029,7 +3131,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 22.7, + "duration_ms": 26.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3057,7 +3159,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.01, + "duration_ms": 26.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3080,7 +3182,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 22.98, + "duration_ms": 25.5, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3108,7 +3210,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 22.64, + "duration_ms": 25.91, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3131,7 +3233,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 22.61, + "duration_ms": 29.64, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3158,7 +3260,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 22.62, + "duration_ms": 30.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3181,7 +3283,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.24, + "duration_ms": 31.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3210,7 +3312,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 23.29, + "duration_ms": 33.98, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -7181,7 +7283,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 152, - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" }, "failures": [], "warnings": [], @@ -8980,10 +9082,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 811, - "other_text_tokens": 871465, + "other_text_tokens": 871624, "estimated_initial_load_tokens": 987, - "estimated_total_text_tokens": 872276, - "relevant_file_count": 367, + "estimated_total_text_tokens": 872435, + "relevant_file_count": 368, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.7 @@ -10692,7 +10794,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T11:10:15Z", + "generated_at": "2026-06-13T11:22:45Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -10851,8 +10953,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5", - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, "compatibility": { "openai": "pass", @@ -10883,7 +10985,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -10907,7 +11009,7 @@ "agent-skills-compatible" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" } ] }, @@ -10930,7 +11032,7 @@ "target_count": 3, "adapter_count": 3, "archive_present": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "archive_entry_count": 490, "failure_count": 0, "warning_count": 0 @@ -11627,12 +11729,12 @@ { "field": "archive_sha256", "from": "", - "to": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "to": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "to": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 1b18e4d..ded5f21 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 152, - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index e006bc6..5bb642a 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `152` -- Package SHA256: `d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5` +- Package SHA256: `af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068` ## Failures diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index ae003a9..1bad241 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -27,9 +27,9 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Runtime Permission Probes v0 for packaged target adapter checks, explicit native-enforcement flags, metadata fallback evidence, and residual permission risks. - Atlas Scope Policy v0 so examples, evolution snapshots, embedded generated skills, and validator fixtures remain visible in the full portfolio report without polluting release-actionable gates. - Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision. -- Review Studio now reaches a clean `ready` decision for the root Meta Skill: `13 / 13` gates pass, world-class score `100`, no blockers, no warnings, and no review actions required. +- Review Studio now avoids over-claiming release readiness when blind A/B adjudication is still pending: the root Meta Skill is in `review` with score `92`, no blockers, two warnings, and explicit actions for Output Lab reviewer adjudication plus waiver handling. -This is still not the final world-class state. Target-native behavior contracts are now explicit and local output-eval command execution is wired, but deeper provider-native execution transforms, real client telemetry capture, installer integration, provider-backed model-executed output eval, real human adjudication decisions, and native runtime permission enforcement remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit and local output-eval command execution is wired, and Review Studio now keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, real client telemetry capture, installer integration, provider-backed model-executed output eval, real human adjudication decisions, and native runtime permission enforcement remain open. ## Coverage Matrix @@ -73,7 +73,7 @@ Next move: add provider-backed model execution for holdout cases and one real mu ### 3. Review Studio is unified and now has reviewer actions plus annotations -The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. After adding root intent context, Atlas scope policy, metadata-only telemetry evidence, runtime permission probe evidence, action source refs, and an empty annotation ledger, the current root report is clean: decision `ready`, score `100`, `13` gates, `0` gate blockers, `0` gate warnings, `0` actions, and `0` open annotation blockers. +The Review Studio page aggregates intent, trigger, output, context, runtime, trust, permission approvals, runtime permission probes, atlas, operations-loop, reviewer waiver, reviewer annotations, registry, and release gates. It exposes current warnings directly and emits `review_actions` for each non-pass gate with a source-fix location, structured `source_refs`, reason, evidence path, and verification command. It now also loads `reports/review_annotations.json`, renders reviewer comments tied to gates and source/report paths, and blocks the page decision when any open blocker annotation exists. The current root report is intentionally not a clean pass: decision `review`, score `92`, `13` gates, `0` blockers, `2` warnings, `2` actions, and `0` open annotation blockers. The warnings are useful evidence: the automated Output Eval Lab is strong, but the blind A/B adjudication still has `5` pending reviewer decisions, so the package should not claim fully reviewed status. Next move: add richer source-line anchors inside generated reports and record real reviewer annotations during the next human review pass. @@ -101,13 +101,13 @@ Next move: add real client or installer permission enforcement integration. | Runtime Permission Probes | `3 / 3` target adapters probed, `0` native-enforcement adapters, `3` explicit metadata fallbacks, `3` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `3 / 3` target adapters present, archive verified, `483` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `483` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `3` adapters readable, `0` failures, `0` warnings | +| Package Verification | `3 / 3` target adapters present, archive verified, `490` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `490` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `3` adapters readable, `0` failures, `0` warnings | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only activation event, adoption `100`, risk band `low`; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | -| Review Waivers | ledger generated; current release has no warning gates to waive, blockers remain non-waivable in v0 | +| Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0 | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | -| Review Studio | decision `ready`, world-class score `100`, `13` gates, `0` blockers, `0` warnings, `0` review actions, `0` open annotation blockers | +| Review Studio | decision `review`, world-class score `92`, `13` gates, `0` blockers, `2` warnings, `2` review actions, `0` open annotation blockers | | IR-first Packaging | `openai`, `claude`, and `generic` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, and semantic parity checks | | Context Budget | initial load `987/1000`, under the production budget | | CI | `make ci-test` target count is now `56` after adding Review Annotations | @@ -116,6 +116,6 @@ Next move: add real client or installer permission enforcement integration. 1. Deepen target-native behavior contracts into provider-native execution and installer integrations. 2. Add native client or installer enforcement for approved high-permission capabilities. -3. Expand Output Eval Lab from local command-runner evidence to provider-backed model holdout and real reviewer-adjudicated cases. +3. Expand Output Eval Lab from local command-runner evidence to provider-backed model holdout and real reviewer-adjudicated cases; record the current blind A/B decisions before claiming fully ready status. 4. Add real reviewer annotation records during the next human review pass. 5. Connect Skill Atlas with real client telemetry and drift history. diff --git a/reports/skill-overview.html b/reports/skill-overview.html index eeced4a..5d99690 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -609,7 +609,7 @@ -
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
+
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 365 个词/字，references 约 14278 个词/字。入口约 365 个词/字，references 约 14278 个词/字。
@@ -641,10 +641,10 @@

指标判读Reading

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
+
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 365 个词/字，references 约 14278 个词/字。入口约 365 个词/字，references 约 14278 个词/字。
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 365 个词/字，references 约 14247 个词/字。入口约 365 个词/字，references 约 14247 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 365 个词/字。SKILL.md 约 365 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 365 个词/字，references 约 14278 个词/字。入口约 365 个词/字，references 约 14278 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
@@ -720,7 +720,7 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 365 个词/字，references 约 14278 个词/字。上下文成本需要补强：入口约 365 个词/字，references 约 14278 个词/字。纳入下一轮修复Fix next
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 58b7545..a595d32 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -96,7 +96,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 365 个词/字，references 约 14247 个词/字。", + "入口约 365 个词/字，references 约 14278 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -200,7 +200,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 365 个词/字，references 约 14247 个词/字。" + "上下文成本需要补强：入口约 365 个词/字，references 约 14278 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -941,7 +941,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 152, - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" }, "skill_atlas": { "skill_count": 12, @@ -975,8 +975,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5", - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "package_sha256": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, "compatibility": { "openai": "pass", @@ -1007,7 +1007,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1023,7 +1023,7 @@ "target_count": 3, "adapter_count": 3, "archive_present": true, - "archive_sha256": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451", + "archive_sha256": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b", "archive_entry_count": 490, "failure_count": 0, "warning_count": 0 @@ -1097,12 +1097,12 @@ { "field": "archive_sha256", "from": "", - "to": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "to": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "to": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" } ] }, diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 77285e1..8933852 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -68,12 +68,12 @@ { "field": "archive_sha256", "from": "", - "to": "6972bb1d72746b6a16c7dbf1ec73f438252756dd850fe474bc3b75af8649d451" + "to": "9208791ccc286c01c0614f5de67ebcff69d90c5be8b09e19ad9617e4059d8d6b" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "d53de57f6593fca3dd937832f5923bd5381250e382631a6b6616e2405de748b5" + "to": "af6eab9b547a289ed5035eed9a9dd54cfdd01459acf1e2ba57d8ed3e2fbab068" } ] }, diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index b76ce52..a6c7a51 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -352,9 +352,9 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = { "verification": "python3 scripts/build_confusion_matrix.py", }, "output-lab": { - "summary": "补足 output eval 的 case 数、file-backed case、near-neighbor case 和 boundary case。", - "why": "没有输出质量证据时，Skill 只能证明会触发，不能证明输出真的更好。", - "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md", + "summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。", + "why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_adjudication.md", "source_paths": [ {"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ["case_id"]}, {"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]}, diff --git a/scripts/review_studio_gates.py b/scripts/review_studio_gates.py index cdd6279..b1609e9 100644 --- a/scripts/review_studio_gates.py +++ b/scripts/review_studio_gates.py @@ -148,8 +148,11 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An execution_recorded_count = int(output_execution_summary.get("recorded_fixture_count", 0) or 0) review_pair_count = int(output_review_summary.get("pair_count", 0) or 0) review_judgment_count = int(output_review_summary.get("judgment_count", 0) or 0) + review_pending_count = int(output_review_summary.get("pending_count", 0) or 0) review_invalid_count = int(output_review_summary.get("invalid_decision_count", 0) or 0) blind_missing = maturity in {"production", "library", "governed"} and (not output_blind or blind_pair_count < case_count) + review_missing = maturity in {"production", "library", "governed"} and case_count > 0 and not output_review + review_pending = maturity in {"production", "library", "governed"} and bool(output_review) and review_pending_count > 0 execution_failed = bool(output_execution) and (not output_execution.get("ok", True) or int(output_execution_summary.get("failure_count", 0) or 0) > 0) review_invalid = bool(output_review) and (not output_review.get("ok", True) or review_invalid_count > 0) output_blocked = ( @@ -159,7 +162,7 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An or execution_failed or review_invalid ) - output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing + output_warn = file_backed == 0 or near_neighbor == 0 or boundary == 0 or blind_missing or review_missing or review_pending if not output: output_status = "warn" output_detail = "output eval scorecard is missing; generate it before production review" @@ -176,6 +179,8 @@ def build_gates(skill_dir: Path, output_html: Path, data: dict[str, dict[str, An else "" ) + (f"; reviewed {review_judgment_count}/{review_pair_count}" if output_review else "") + + (f"; review pending {review_pending_count}" if review_pending else "") + + ("; review adjudication missing" if review_missing else "") ) gates.append( gate( @@ -509,4 +514,3 @@ def weighted_score(gates: list[dict[str, str]]) -> int: elif item["status"] == "warn": earned += weight * 0.6 return int(round(earned / total * 100)) if total else 0 - diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 1048c10..9e3974d 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -229,26 +229,27 @@ def main() -> None: payload = json.loads(proc.stdout) assert payload["ok"], payload assert payload["schema_version"] == "2.0", payload - assert payload["summary"]["decision"] == "ready", payload + assert payload["summary"]["decision"] == "review", payload assert payload["summary"]["gate_count"] == 13, payload - assert payload["summary"]["world_class_score"] == 100, payload - assert payload["summary"]["warning_count"] == 0, payload + assert payload["summary"]["world_class_score"] == 92, payload + assert payload["summary"]["warning_count"] == 2, payload assert payload["summary"]["blocker_count"] == 0, payload - assert payload["summary"]["action_count"] == 0, payload + assert payload["summary"]["action_count"] == 2, payload assert payload["summary"]["annotation_count"] == 0, payload assert payload["summary"]["open_annotation_blocker_count"] == 0, payload assert payload["summary"]["action_count"] == payload["summary"]["warning_count"] + payload["summary"]["blocker_count"], payload - assert payload["review_actions"] == [], payload + assert {item["gate_key"] for item in payload["review_actions"]} == {"output-lab", "review-waivers"}, payload gate_keys = {item["key"] for item in payload["gates"]} assert {"intent-canvas", "trigger-lab", "output-lab", "runtime-matrix", "trust-report", "permission-gates", "permission-runtime", "skill-atlas", "operations-loop", "review-waivers", "registry-audit", "release-notes"} <= gate_keys, payload output_gate = next(item for item in payload["gates"] if item["key"] == "output-lab") - assert output_gate["status"] == "pass", output_gate + assert output_gate["status"] == "warn", output_gate assert "5/5 cases" in output_gate["detail"], output_gate assert "file-backed 1" in output_gate["detail"], output_gate assert "blind A/B 5" in output_gate["detail"], output_gate assert "exec 10" in output_gate["detail"], output_gate assert "model 0" in output_gate["detail"], output_gate assert "reviewed 0/5" in output_gate["detail"], output_gate + assert "review pending 5" in output_gate["detail"], output_gate release_gate = next(item for item in payload["gates"] if item["key"] == "release-notes") assert "upgrade minor declared / minor recommended" in release_gate["detail"], release_gate assert "reports/upgrade_check.json" in release_gate["evidence"], release_gate @@ -279,8 +280,8 @@ def main() -> None: assert "risk low" in operations_gate["detail"], operations_gate assert "reports/adoption_drift_report.json" in operations_gate["evidence"], operations_gate waivers_gate = next(item for item in payload["gates"] if item["key"] == "review-waivers") - assert waivers_gate["status"] == "pass", waivers_gate - assert "cover current warnings" in waivers_gate["detail"], waivers_gate + assert waivers_gate["status"] == "warn", waivers_gate + assert "warning gates still need reviewer decision" in waivers_gate["detail"], waivers_gate assert "reports/review_waivers.json" in waivers_gate["evidence"], waivers_gate assert output_html.exists(), output_html assert output_json.exists(), output_json @@ -301,7 +302,7 @@ def main() -> None: assert full_payload["data"]["runtime_permissions"]["summary"]["metadata_fallback_count"] == 3, full_payload["data"]["runtime_permissions"] assert full_payload["evidence_paths"]["runtime_permissions"] == "reports/runtime_permission_probes.md", full_payload["evidence_paths"] action_keys = {item["gate_key"] for item in full_payload["review_actions"]} - assert action_keys == set(), full_payload["review_actions"] + assert action_keys == {"output-lab", "review-waivers"}, full_payload["review_actions"] assert full_payload["data"]["atlas"]["summary"]["actionable_route_collision_count"] == 0, full_payload["data"]["atlas"] assert full_payload["data"]["atlas"]["summary"]["non_actionable_issue_count"] >= 1, full_payload["data"]["atlas"] synthetic_actions = review_studio.build_review_actions( @@ -338,7 +339,8 @@ def main() -> None: assert "Review Studio 2.0" in html, html[:400] assert "审查闸门" in html, html[:1200] assert "修复动作" in html, html[:3000] - assert "当前没有 blocker 或 warning" in html, html[:9000] + assert "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。" in html, html[:9000] + assert "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。" in html, html[:9000] assert "审查批注" in html, html[:9000] assert "当前没有 reviewer 批注" in html, html[:9000] assert "输出实验" in html, html[:2000]