diff --git a/references/output-eval-method.md b/references/output-eval-method.md index 59b43f14..fbe7fa74 100644 --- a/references/output-eval-method.md +++ b/references/output-eval-method.md @@ -106,6 +106,8 @@ The adjudication report writes: When no reviewer decisions exist, the report should say the cases are pending and Review Studio should link to the decisions template. Do not count pending cases as human agreement. Only a real `winner_variant` of `A` or `B` should contribute to agreement rate, disagreement count, and reviewer judgment count. +The adjudication report must preserve blind-review integrity: pending and invalid decisions should show the expected winner as hidden. Only reveal `expected_winner_variant` after a valid reviewer decision exists for that case. + ## Anti-Overfitting Keep a small public smoke set and a separate holdout set. Rotate real failures into the taxonomy instead of editing only the prompt that failed. Add near-neighbor cases whenever the output looks good but the boundary is still unclear. diff --git a/registry/index.json b/registry/index.json index 052a2d49..191a68d6 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index b0970011..a5d17732 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/context_budget.json b/reports/context_budget.json index f9c3c9af..1af2b908 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 910665, + "other_text_tokens": 911753, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 911416, - "relevant_file_count": 388, + "estimated_total_text_tokens": 912504, + "relevant_file_count": 390, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/output_review_adjudication.json b/reports/output_review_adjudication.json index 0d58d328..178315f9 100644 --- a/reports/output_review_adjudication.json +++ b/reports/output_review_adjudication.json @@ -8,6 +8,8 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, "failure_count": 0 @@ -26,7 +28,8 @@ { "case_id": "skill-package-contract", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -35,7 +38,8 @@ { "case_id": "output-eval-expectation", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -44,7 +48,8 @@ { "case_id": "ir-before-packaging", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -53,7 +58,8 @@ { "case_id": "near-neighbor-boundary", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -62,7 +68,8 @@ { "case_id": "file-backed-governed-package", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", diff --git a/reports/output_review_adjudication.md b/reports/output_review_adjudication.md index 6c12de42..48d85c16 100644 --- a/reports/output_review_adjudication.md +++ b/reports/output_review_adjudication.md @@ -7,20 +7,23 @@ This report adjudicates reviewer choices from the blind A/B output review pack a - Pending: `5` - Agreement rate: `n/a` - Invalid decisions: `0` +- Answer keys revealed: `0` +- Pending/invalid answers hidden: `5` No reviewer decisions recorded yet. Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication. +Expected winners stay hidden until a valid reviewer decision is recorded. ## Case Adjudication | Case | Reviewer | Expected | Status | Confidence | Reason | | --- | --- | --- | --- | ---: | --- | -| skill-package-contract | pending | A | pending | | | -| output-eval-expectation | pending | A | pending | | | -| ir-before-packaging | pending | B | pending | | | -| near-neighbor-boundary | pending | B | pending | | | -| file-backed-governed-package | pending | B | pending | | | +| skill-package-contract | pending | hidden | pending | | | +| output-eval-expectation | pending | hidden | pending | | | +| ir-before-packaging | pending | hidden | pending | | | +| near-neighbor-boundary | pending | hidden | pending | | | +| file-backed-governed-package | pending | hidden | pending | | | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 4c9e15dc..c8f8b3e3 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index e4ab6733..12ea6e81 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20` +- Archive SHA256: `a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index e731b259..225e3897 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 89523c53..aebc1f2d 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb` -- Archive SHA256: `23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20` +- Package SHA256: `3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131` +- Archive SHA256: `a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index c7d40e8d..250195e5 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -247,7 +247,7 @@

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 69 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 69 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -289,7 +289,7 @@

盲评包

Pair Count
5
Answer Key Separate
是
With Skill Hidden Count
5
-

审定报告

Pair Count
5
Judgment Count
0
待审
5
Agreement Count
0
Disagreement Count
0
Invalid Decision Count
0
+

审定报告

Pair Count
5
Judgment Count
0
待审
5
Agreement Count
0
Disagreement Count
0
Invalid Decision Count
0
答案揭示
0
答案隐藏
5
@@ -313,7 +313,7 @@
-

信任报告

Secret
0
脚本数
69
网络脚本
3
Help 失败
0
包体哈希
ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb
+

信任报告

Secret
0
脚本数
69
网络脚本
3
Help 失败
0
包体哈希
3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -333,8 +333,8 @@
-

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

-

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

+

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
@@ -344,12 +344,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
494
失败数
0
警告数
0
归档哈希
23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
494
失败数
0
警告数
0
归档哈希
a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6
diff --git a/reports/review-studio.json b/reports/review-studio.json index 2e98744b..0c764e24 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -91,7 +91,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low", + "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low", "evidence": "reports/adoption_drift_report.json", "link": "adoption_drift_report.md" }, @@ -355,7 +355,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 350 个词/字，references 约 14610 个词/字。", + "入口约 350 个词/字，references 约 14640 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -459,7 +459,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。" + "上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -1130,6 +1130,8 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, "failure_count": 0 @@ -1226,7 +1228,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" }, "skill_atlas": { "skill_count": 12, @@ -1264,8 +1266,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, "compatibility": { "openai": "pass", @@ -1296,7 +1298,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1312,7 +1314,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 @@ -1387,12 +1389,12 @@ { "field": "archive_sha256", "from": "", - "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" } ] }, @@ -1409,14 +1411,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1425,7 +1427,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {} }, @@ -3122,7 +3124,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 33.11, + "duration_ms": 31.8, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3150,7 +3152,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.55, + "duration_ms": 31.63, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3173,7 +3175,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.73, + "duration_ms": 30.68, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3201,7 +3203,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.84, + "duration_ms": 32.69, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3224,7 +3226,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.53, + "duration_ms": 32.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3252,7 +3254,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.84, + "duration_ms": 34.03, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3275,7 +3277,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.54, + "duration_ms": 34.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3302,7 +3304,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.69, + "duration_ms": 34.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3325,7 +3327,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.54, + "duration_ms": 33.14, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3354,7 +3356,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.91, + "duration_ms": 36.79, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3591,6 +3593,8 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, "failure_count": 0 @@ -3609,7 +3613,8 @@ { "case_id": "skill-package-contract", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -3618,7 +3623,8 @@ { "case_id": "output-eval-expectation", "status": "pending", - "expected_winner_variant": "A", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -3627,7 +3633,8 @@ { "case_id": "ir-before-packaging", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -3636,7 +3643,8 @@ { "case_id": "near-neighbor-boundary", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -3645,7 +3653,8 @@ { "case_id": "file-backed-governed-package", "status": "pending", - "expected_winner_variant": "B", + "expected_winner_variant": "", + "expected_revealed": false, "reviewer_winner_variant": "", "confidence": null, "reason": "", @@ -8215,7 +8224,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" }, "failures": [], "warnings": [], @@ -10044,10 +10053,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 910655, + "other_text_tokens": 911753, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 911406, - "relevant_file_count": 388, + "estimated_total_text_tokens": 912504, + "relevant_file_count": 390, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -11043,8 +11052,8 @@ "report": "reports/adoption_drift_report.json", "risk_band": "low", "event_count": 1, - "adoption_sample_count": 0, - "adoption_rate": 0, + "adoption_sample_count": 1, + "adoption_rate": 100.0, "candidate_count": 0 } }, @@ -11875,7 +11884,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T12:24:01Z", + "generated_at": "2026-06-13T12:26:25Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -11899,14 +11908,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -11915,7 +11924,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {} }, @@ -11923,29 +11932,29 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 0, - "accepted": 0, + "adoption_events": 1, + "accepted": 1, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 0 + "adoption_rate": 100.0 } ], "next_iteration_candidates": [], "recent_events": [ { - "event": "review_event", + "event": "skill_activation", "skill": "yao-meta-skill", "version": "1.1.0", - "activation_type": "manual", - "outcome": "reviewed", + "activation_type": "explicit", + "outcome": "accepted", "failure_type": "none", - "timestamp": "2026-06-13T12:00:00Z" + "timestamp": "2026-06-13T10:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "reports/telemetry_events.jsonl", + "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -11994,7 +12003,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "reports/review_annotations_input.json", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -12037,8 +12046,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, "compatibility": { "openai": "pass", @@ -12069,7 +12078,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -12094,7 +12103,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" } ] }, @@ -12117,7 +12126,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 @@ -12976,12 +12985,12 @@ { "field": "archive_sha256", "from": "", - "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" } ] }, diff --git a/reports/runtime_permission_probes.json b/reports/runtime_permission_probes.json index 6d6254fb..a989e325 100644 --- a/reports/runtime_permission_probes.json +++ b/reports/runtime_permission_probes.json @@ -2,7 +2,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "package_dir": "tests/tmp_review_studio/dist", + "package_dir": "dist", "expected_capabilities": [ "file_write", "network", @@ -22,7 +22,7 @@ { "target": "openai", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json", + "adapter": "dist/targets/openai/adapter.json", "permission_model": "metadata-only", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -107,7 +107,7 @@ { "target": "claude", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json", + "adapter": "dist/targets/claude/adapter.json", "permission_model": "neutral-source-plus-adapter", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -177,7 +177,7 @@ { "target": "generic", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json", + "adapter": "dist/targets/generic/adapter.json", "permission_model": "agent-skills-compatible-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, @@ -247,7 +247,7 @@ { "target": "vscode", "status": "pass", - "adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json", + "adapter": "dist/targets/vscode/adapter.json", "permission_model": "vscode-workspace-trust-plus-metadata", "native_enforcement": false, "metadata_fallback_explicit": true, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index aa9cceff..a27cbe0d 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index a9c9838c..88f12449 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `153` -- Package SHA256: `ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb` +- Package SHA256: `3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131` ## Failures diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 78a3ab23..4629ca8e 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -29,9 +29,10 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision. - Review Studio now avoids over-claiming release readiness when blind A/B adjudication is still pending: the root Meta Skill is in `review` with score `92`, no blockers, two warnings, and explicit actions for Output Lab reviewer adjudication plus waiver handling. - Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning. +- Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case. - Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue. -This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, a provider-backed output runner exists, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, real client telemetry capture, installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open. +This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, real client telemetry capture, installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open. ## Coverage Matrix @@ -95,7 +96,7 @@ Next move: add real client or installer permission enforcement integration. | Gate | Current Result | | --- | --- | -| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, and `0 / 5` reviewer decisions pending | +| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | | Trust | `0` secret findings, `1` pinned dependency file, `9` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `60 / 60` CLI help smoke checks passing across `69` scripts, source-contract hash scope explicit | diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 80b1a195..3aa8ad7b 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -609,7 +609,7 @@
-
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 350 个词/字，references 约 14610 个词/字。入口约 350 个词/字，references 约 14610 个词/字。
+
完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
上下文成本42入口约 350 个词/字，references 约 14640 个词/字。入口约 350 个词/字，references 约 14640 个词/字。
@@ -641,10 +641,10 @@

指标判读Reading

先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.

-
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 350 个词/字，references 约 14610 个词/字。入口约 350 个词/字，references 约 14610 个词/字。
+
  1. 稳定Stable完整度100SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  2. 稳定Stable触发清晰100frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  3. 稳定Stable证据充分100已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  4. 稳定Stable可维护性100SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  5. 稳定Stable可迁移性100agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  6. 关注Watch上下文成本42入口约 350 个词/字，references 约 14640 个词/字。入口约 350 个词/字，references 约 14640 个词/字。
-
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 350 个词/字，references 约 14610 个词/字。入口约 350 个词/字，references 约 14610 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
+
完整度100
  • SKILL.md 已存在，是 Skill 的入口。SKILL.md 已存在，是 Skill 的入口。
  • README.md 已存在，便于人工阅读。README.md 已存在，便于人工阅读。
  • agents/interface.yaml 已存在，便于跨平台适配。agents/interface.yaml 已存在，便于跨平台适配。
触发清晰100
  • frontmatter description 已存在，具备基础路由面。frontmatter description 已存在，具备基础路由面。
  • description 有足够长度说明任务边界。description 有足够长度说明任务边界。
  • description 已包含使用场景或排除边界信号。description 已包含使用场景或排除边界信号。
证据充分100
  • 已生成 20 / 20 类报告证据。已生成 20 / 20 类报告证据。
  • skill-ir.json 已存在。skill-ir.json 已存在。
  • compiled_targets.json 已存在。compiled_targets.json 已存在。
可维护性100
  • SKILL.md 约 350 个词/字。SKILL.md 约 350 个词/字。
  • 入口文件保持克制，可维护性较好。入口文件保持克制，可维护性较好。
  • references/ 已承载扩展指导。references/ 已承载扩展指导。
可迁移性100
  • agents/interface.yaml 已存在。agents/interface.yaml 已存在。
  • manifest.json 已存在。manifest.json 已存在。
  • 目标平台或 adapter target 已声明。目标平台或 adapter target 已声明。
上下文成本42
  • 入口约 350 个词/字，references 约 14640 个词/字。入口约 350 个词/字，references 约 14640 个词/字。
  • 分数越高代表上下文成本越低。分数越高代表上下文成本越低。
  • 上下文成本偏高，建议压缩入口或拆分 references。上下文成本偏高，建议压缩入口或拆分 references。
@@ -720,7 +720,7 @@
- +
类型Type证据Evidence建议Action
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。纳入下一轮修复Fix next
强项Strength触发面保持精简，并锚定在 frontmatter description。The trigger surface stays lean and anchored in the frontmatter description.保留并复用Keep
强项Strength已生成 Skill IR，核心语义可先于平台打包被审查和迁移。已生成 Skill IR，核心语义可先于平台打包被审查和迁移。保留并复用Keep
强项Strength已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。保留并复用Keep
缺口Gap上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。纳入下一轮修复Fix next
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index ec62842a..32b80d7b 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -96,7 +96,7 @@ "label": "上下文成本", "score": 42, "reasons": [ - "入口约 350 个词/字，references 约 14610 个词/字。", + "入口约 350 个词/字，references 约 14640 个词/字。", "分数越高代表上下文成本越低。", "上下文成本偏高，建议压缩入口或拆分 references。" ] @@ -200,7 +200,7 @@ "已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。" ], "gaps": [ - "上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。" + "上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。" ], "recommendations": [ "先改触发边界，再扩展工作流。", @@ -871,6 +871,8 @@ "agreement_count": 0, "disagreement_count": 0, "invalid_decision_count": 0, + "answer_revealed_count": 0, + "pending_answer_hidden_count": 5, "agreement_rate": null, "needs_review": true, "failure_count": 0 @@ -967,7 +969,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 153, - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" }, "skill_atlas": { "skill_count": 12, @@ -1005,8 +1007,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb", - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, "compatibility": { "openai": "pass", @@ -1037,7 +1039,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1053,7 +1055,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20", + "archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6", "archive_entry_count": 494, "failure_count": 0, "warning_count": 0 @@ -1128,12 +1130,12 @@ { "field": "archive_sha256", "from": "", - "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" } ] }, @@ -1150,14 +1152,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1166,7 +1168,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {} }, diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index b3043f1a..51797264 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20" + "to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb" + "to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131" } ] }, @@ -90,7 +90,7 @@ "artifacts": { "previous_package": "registry/examples/yao-meta-skill-1.0.0.json", "current_package": "reports/registry_audit.json", - "json": "tests/tmp_review_studio/upgrade_check.json", - "markdown": "tests/tmp_review_studio/upgrade_check.md" + "json": "reports/upgrade_check.json", + "markdown": "reports/upgrade_check.md" } } diff --git a/scripts/adjudicate_output_review.py b/scripts/adjudicate_output_review.py index 759bc530..1200481d 100644 --- a/scripts/adjudicate_output_review.py +++ b/scripts/adjudicate_output_review.py @@ -137,7 +137,8 @@ def adjudicate_pair( { "case_id": case_id, "status": "pending", - "expected_winner_variant": expected, + "expected_winner_variant": "", + "expected_revealed": False, "reviewer_winner_variant": "", "confidence": None, "reason": "", @@ -156,7 +157,8 @@ def adjudicate_pair( { "case_id": case_id, "status": "pending", - "expected_winner_variant": expected, + "expected_winner_variant": "", + "expected_revealed": False, "reviewer_winner_variant": "", "confidence": confidence, "reason": reason, @@ -164,7 +166,9 @@ def adjudicate_pair( }, failures, ) - if reviewer not in {"A", "B"}: + if expected not in {"A", "B"}: + status = "invalid" + elif reviewer not in {"A", "B"}: failures.append(f"{case_id}: winner_variant must be A or B") status = "invalid" elif confidence_failure: @@ -172,11 +176,13 @@ def adjudicate_pair( status = "invalid" else: status = "match" if reviewer == expected else "disagree" + expected_revealed = status in {"match", "disagree"} return ( { "case_id": case_id, "status": status, - "expected_winner_variant": expected, + "expected_winner_variant": expected if expected_revealed else "", + "expected_revealed": expected_revealed, "reviewer_winner_variant": reviewer, "confidence": confidence, "reason": reason, @@ -193,6 +199,7 @@ def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, agreement_count = sum(1 for item in pairs if item["status"] == "match") disagreement_count = sum(1 for item in pairs if item["status"] == "disagree") invalid_decision_count = sum(1 for item in pairs if item["status"] == "invalid") + answer_revealed_count = sum(1 for item in pairs if item.get("expected_revealed")) agreement_rate = round(agreement_count / judgment_count * 100, 2) if judgment_count else None return { "pair_count": pair_count, @@ -201,6 +208,8 @@ def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str, "agreement_count": agreement_count, "disagreement_count": disagreement_count, "invalid_decision_count": invalid_decision_count, + "answer_revealed_count": answer_revealed_count, + "pending_answer_hidden_count": sum(1 for item in pairs if item["status"] in {"pending", "invalid"} and not item.get("expected_revealed")), "agreement_rate": agreement_rate, "needs_review": pending_count > 0, "failure_count": len(failures), @@ -219,6 +228,8 @@ def render_markdown(payload: dict[str, Any]) -> str: f"- Pending: `{summary['pending_count']}`", f"- Agreement rate: `{summary['agreement_rate'] if summary['agreement_rate'] is not None else 'n/a'}`", f"- Invalid decisions: `{summary['invalid_decision_count']}`", + f"- Answer keys revealed: `{summary['answer_revealed_count']}`", + f"- Pending/invalid answers hidden: `{summary['pending_answer_hidden_count']}`", "", ] if summary["judgment_count"] == 0: @@ -227,6 +238,7 @@ def render_markdown(payload: dict[str, Any]) -> str: "No reviewer decisions recorded yet.", "", "Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.", + "Expected winners stay hidden until a valid reviewer decision is recorded.", "", ] ) @@ -241,9 +253,10 @@ def render_markdown(payload: dict[str, Any]) -> str: for item in payload["pairs"]: confidence = "" if item.get("confidence") is None else str(item["confidence"]) reason = str(item.get("reason", "")).replace("|", "\\|") or "" + expected = item.get("expected_winner_variant", "") if item.get("expected_revealed") else "hidden" lines.append( f"| {item['case_id']} | {item.get('reviewer_winner_variant', '') or 'pending'} | " - f"{item.get('expected_winner_variant', '') or 'missing'} | {item['status']} | {confidence} | {reason} |" + f"{expected} | {item['status']} | {confidence} | {reason} |" ) if payload.get("failures"): lines.extend(["", "## Failures", ""]) diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index 3bd2a1b0..0ce7508d 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -665,7 +665,16 @@ def render_html(report: dict[str, Any]) -> str: ) review_panel = render_kv_grid( output_review_summary, - ["pair_count", "judgment_count", "pending_count", "agreement_count", "disagreement_count", "invalid_decision_count"], + [ + "pair_count", + "judgment_count", + "pending_count", + "agreement_count", + "disagreement_count", + "invalid_decision_count", + "answer_revealed_count", + "pending_answer_hidden_count", + ], "review adjudication report missing", ) conformance_panel = render_kv_grid( diff --git a/scripts/review_studio_formatting.py b/scripts/review_studio_formatting.py index f7195fea..736b08b7 100644 --- a/scripts/review_studio_formatting.py +++ b/scripts/review_studio_formatting.py @@ -19,6 +19,7 @@ LABELS = { "archive_entry_count": "Zip 条目", "archive_present": "归档存在", "archive_sha256": "归档哈希", + "answer_revealed_count": "答案揭示", "baseline_pass_rate": "Baseline", "breaking_change_count": "破坏变更", "case_count": "案例数", @@ -46,6 +47,7 @@ LABELS = { "package_sha256": "包体哈希", "pass_count": "通过数", "pending_count": "待审", + "pending_answer_hidden_count": "答案隐藏", "recorded_fixture_count": "记录样本", "recommended_bump": "建议版本", "residual_risk_count": "残余风险", diff --git a/tests/verify_output_review_adjudication.py b/tests/verify_output_review_adjudication.py index 93ac4fa8..dcab76e2 100644 --- a/tests/verify_output_review_adjudication.py +++ b/tests/verify_output_review_adjudication.py @@ -75,7 +75,13 @@ def main() -> None: assert pending_payload["summary"]["pending_count"] == 5, pending_payload assert pending_payload["summary"]["agreement_rate"] is None, pending_payload assert pending_payload["summary"]["needs_review"], pending_payload + assert pending_payload["summary"]["answer_revealed_count"] == 0, pending_payload + assert pending_payload["summary"]["pending_answer_hidden_count"] == 5, pending_payload + assert all(not item["expected_winner_variant"] and not item["expected_revealed"] for item in pending_payload["pairs"]), pending_payload assert "No reviewer decisions recorded yet" in pending_md.read_text(encoding="utf-8"), pending_md + pending_text = pending_md.read_text(encoding="utf-8") + assert "| skill-package-contract | pending | hidden | pending |" in pending_text, pending_text + assert "| skill-package-contract | pending | A | pending |" not in pending_text, pending_text template_path = tmp_root / "output_review_decisions.json" template_proc = run( @@ -139,7 +145,10 @@ def main() -> None: assert filled_payload["summary"]["pending_count"] == 0, filled_payload assert filled_payload["summary"]["agreement_count"] == 5, filled_payload assert filled_payload["summary"]["agreement_rate"] == 100.0, filled_payload + assert filled_payload["summary"]["answer_revealed_count"] == 5, filled_payload + assert filled_payload["summary"]["pending_answer_hidden_count"] == 0, filled_payload assert all(item["status"] == "match" for item in filled_payload["pairs"]), filled_payload + assert all(item["expected_winner_variant"] in {"A", "B"} and item["expected_revealed"] for item in filled_payload["pairs"]), filled_payload invalid = { "schema_version": "1.0", @@ -176,6 +185,9 @@ def main() -> None: invalid_payload = json.loads(invalid_proc.stdout) assert not invalid_payload["ok"], invalid_payload assert invalid_payload["summary"]["invalid_decision_count"] == 1, invalid_payload + assert invalid_payload["summary"]["answer_revealed_count"] == 0, invalid_payload + assert invalid_payload["pairs"][0]["expected_winner_variant"] == "", invalid_payload + assert invalid_payload["pairs"][0]["expected_revealed"] is False, invalid_payload assert invalid_payload["failures"], invalid_payload print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 94433fb3..d39c0076 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -300,6 +300,9 @@ def main() -> None: assert full_payload["data"]["output_execution"]["summary"]["recorded_fixture_count"] == 0, full_payload["data"]["output_execution"] assert full_payload["data"]["output_execution"]["summary"]["model_executed_count"] == 0, full_payload["data"]["output_execution"] assert full_payload["data"]["output_review_adjudication"]["summary"]["pending_count"] == 5, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["answer_revealed_count"] == 0, full_payload["data"]["output_review_adjudication"] + assert full_payload["data"]["output_review_adjudication"]["summary"]["pending_answer_hidden_count"] == 5, full_payload["data"]["output_review_adjudication"] + assert all(not item["expected_revealed"] for item in full_payload["data"]["output_review_adjudication"]["pairs"]), full_payload["data"]["output_review_adjudication"] assert full_payload["data"]["review_annotations"]["summary"]["annotation_count"] == 0, full_payload["data"]["review_annotations"] assert full_payload["data"]["compiled_targets"]["summary"]["target_count"] >= 4, full_payload["data"]["compiled_targets"] assert full_payload["data"]["compiled_targets"]["summary"]["block_count"] == 0, full_payload["data"]["compiled_targets"] @@ -355,6 +358,8 @@ def main() -> None: assert "执行证据" in html, html assert "盲评包" in html, html[:5000] assert "审定报告" in html, html + assert "答案揭示" in html, html + assert "答案隐藏" in html, html assert "注册审计" in html, html[:3000] assert "包体验证" in html, html[:5000] assert "Upgrade" in html, html[:6000]