frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

fix: preserve blind review answer integrity

Browse Source
This commit is contained in:
yaojingang
2026-06-13 20:44:46 +08:00
parent 74541dd689
commit e8cebf93bb
24 changed files with 192 additions and 127 deletions
Download Patch File Download Diff File Expand all files Collapse all files
references/output-eval-method.md
+2
View File
@@ -106,6 +106,8 @@ The adjudication report writes:
When no reviewer decisions exist, the report should say the cases are pending and Review Studio should link to the decisions template. Do not count pending cases as human agreement. Only a real `winner_variant` of `A` or `B` should contribute to agreement rate, disagreement count, and reviewer judgment count.
The adjudication report must preserve blind-review integrity: pending and invalid decisions should show the expected winner as hidden. Only reveal `expected_winner_variant` after a valid reviewer decision exists for that case.
## Anti-Overfitting
Keep a small public smoke set and a separate holdout set. Rotate real failures into the taxonomy instead of editing only the prompt that failed. Add near-neighbor cases whenever the output looks good but the boundary is still unclear.
registry/index.json
+1 -1
View File
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
}
]
}
registry/packages/yao-meta-skill.json
+3 -3
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb",
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
"compatibility": {
"openai": "pass",
@@ -48,7 +48,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
reports/context_budget.json
+3 -3
View File
@@ -6,10 +6,10 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 751,
"other_text_tokens": 910665,
"other_text_tokens": 911753,
"estimated_initial_load_tokens": 944,
"estimated_total_text_tokens": 911416,
"relevant_file_count": 388,
"estimated_total_text_tokens": 912504,
"relevant_file_count": 390,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 137.7
reports/output_review_adjudication.json
+12 -5
View File
@@ -8,6 +8,8 @@
"agreement_count": 0,
"disagreement_count": 0,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"pending_answer_hidden_count": 5,
"agreement_rate": null,
"needs_review": true,
"failure_count": 0
@@ -26,7 +28,8 @@
{
"case_id": "skill-package-contract",
"status": "pending",
"expected_winner_variant": "A",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -35,7 +38,8 @@
{
"case_id": "output-eval-expectation",
"status": "pending",
"expected_winner_variant": "A",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -44,7 +48,8 @@
{
"case_id": "ir-before-packaging",
"status": "pending",
"expected_winner_variant": "B",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -53,7 +58,8 @@
{
"case_id": "near-neighbor-boundary",
"status": "pending",
"expected_winner_variant": "B",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -62,7 +68,8 @@
{
"case_id": "file-backed-governed-package",
"status": "pending",
"expected_winner_variant": "B",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
reports/output_review_adjudication.md
+8 -5
View File
@@ -7,20 +7,23 @@ This report adjudicates reviewer choices from the blind A/B output review pack a
- Pending: `5`
- Agreement rate: `n/a`
- Invalid decisions: `0`
- Answer keys revealed: `0`
- Pending/invalid answers hidden: `5`
No reviewer decisions recorded yet.
Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.
Expected winners stay hidden until a valid reviewer decision is recorded.
## Case Adjudication
| Case | Reviewer | Expected | Status | Confidence | Reason |
| --- | --- | --- | --- | ---: | --- |
| skill-package-contract | pending | A | pending | | |
| output-eval-expectation | pending | A | pending | | |
| ir-before-packaging | pending | B | pending | | |
| near-neighbor-boundary | pending | B | pending | | |
| file-backed-governed-package | pending | B | pending | | |
| skill-package-contract | pending | hidden | pending | | |
| output-eval-expectation | pending | hidden | pending | | |
| ir-before-packaging | pending | hidden | pending | | |
| near-neighbor-boundary | pending | hidden | pending | | |
| file-backed-governed-package | pending | hidden | pending | | |
## Next Fixes
reports/package_verification.json
+1 -1
View File
@@ -8,7 +8,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"archive_entry_count": 494,
"failure_count": 0,
"warning_count": 0
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20`
- Archive SHA256: `a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6`
- Failures: `0`
- Warnings: `0`
reports/registry_audit.json
+4 -4
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb",
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
"compatibility": {
"openai": "pass",
@@ -53,7 +53,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb`
- Archive SHA256: `23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20`
- Package SHA256: `3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131`
- Archive SHA256: `a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6`
- Install simulated: `True`
## Compatibility
reports/review-studio.html
+7 -7
View File
@@ -247,7 +247,7 @@
<section>
<h2>审查闸门</h2>
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 69 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 69 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
</section>
<div class="twocol">
@@ -289,7 +289,7 @@
<section class="twocol">
<div class="panel"><h2>盲评包</h2><dl class='kv-grid'><div><dt>Pair Count</dt><dd>5</dd></div><div><dt>Answer Key Separate</dt><dd>是</dd></div><div><dt>With Skill Hidden Count</dt><dd>5</dd></div></dl></div>
<div class="panel"><h2>审定报告</h2><dl class='kv-grid'><div><dt>Pair Count</dt><dd>5</dd></div><div><dt>Judgment Count</dt><dd>0</dd></div><div><dt>待审</dt><dd>5</dd></div><div><dt>Agreement Count</dt><dd>0</dd></div><div><dt>Disagreement Count</dt><dd>0</dd></div><div><dt>Invalid Decision Count</dt><dd>0</dd></div></dl></div>
<div class="panel"><h2>审定报告</h2><dl class='kv-grid'><div><dt>Pair Count</dt><dd>5</dd></div><div><dt>Judgment Count</dt><dd>0</dd></div><div><dt>待审</dt><dd>5</dd></div><div><dt>Agreement Count</dt><dd>0</dd></div><div><dt>Disagreement Count</dt><dd>0</dd></div><div><dt>Invalid Decision Count</dt><dd>0</dd></div><div><dt>答案揭示</dt><dd>0</dd></div><div><dt>答案隐藏</dt><dd>5</dd></div></dl></div>
</section>
<section class="twocol">
@@ -313,7 +313,7 @@
</section>
<section id="trust" class="twocol">
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>69</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb</code></dd></div></dl></div>
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>69</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131</code></dd></div></dl></div>
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
</section>
@@ -333,8 +333,8 @@
</section>
<section id="telemetry" class="twocol">
<div class="panel"><h2>运营回路</h2><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p></div>
<div class="panel"><h2>漂移信号</h2><dl class='kv-grid'><div><dt>事件数</dt><dd>1</dd></div><div><dt>采用率</dt><dd>0</dd></div><div><dt>漏触发</dt><dd>0</dd></div><div><dt>Bad Output Count</dt><dd>0</dd></div><div><dt>风险带</dt><dd>low</dd></div></dl></div>
<div class="panel"><h2>运营回路</h2><p>1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low</p></div>
<div class="panel"><h2>漂移信号</h2><dl class='kv-grid'><div><dt>事件数</dt><dd>1</dd></div><div><dt>采用率</dt><dd>100</dd></div><div><dt>漏触发</dt><dd>0</dd></div><div><dt>Bad Output Count</dt><dd>0</dd></div><div><dt>风险带</dt><dd>low</dd></div></dl></div>
</section>
<section id="waivers" class="twocol">
@@ -344,12 +344,12 @@
<section id="registry" class="twocol">
<div class="panel"><h2>注册审计</h2><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters</p></div>
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20</code></dd></div></dl></div>
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6</code></dd></div></dl></div>
</section>
<section id="release" class="twocol">
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p></div>
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>494</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20</code></dd></div></dl></div>
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>494</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6</code></dd></div></dl></div>
</section>
</main>
</body>
reports/review-studio.json
+67 -58
View File
@@ -91,7 +91,7 @@
"key": "operations-loop",
"label": "运营回路",
"status": "pass",
"detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low",
"detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low",
"evidence": "reports/adoption_drift_report.json",
"link": "adoption_drift_report.md"
},
@@ -355,7 +355,7 @@
"label": "上下文成本",
"score": 42,
"reasons": [
"入口约 350 个词/字，references 约 14610 个词/字。",
"入口约 350 个词/字，references 约 14640 个词/字。",
"分数越高代表上下文成本越低。",
"上下文成本偏高，建议压缩入口或拆分 references。"
]
@@ -459,7 +459,7 @@
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。"
],
"gaps": [
"上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。"
"上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。"
],
"recommendations": [
"先改触发边界，再扩展工作流。",
@@ -1130,6 +1130,8 @@
"agreement_count": 0,
"disagreement_count": 0,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"pending_answer_hidden_count": 5,
"agreement_rate": null,
"needs_review": true,
"failure_count": 0
@@ -1226,7 +1228,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 153,
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
},
"skill_atlas": {
"skill_count": 12,
@@ -1264,8 +1266,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb",
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
"compatibility": {
"openai": "pass",
@@ -1296,7 +1298,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -1312,7 +1314,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"archive_entry_count": 494,
"failure_count": 0,
"warning_count": 0
@@ -1387,12 +1389,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
}
]
},
@@ -1409,14 +1411,14 @@
"ok": true,
"summary": {
"event_count": 1,
"adoption_sample_count": 0,
"activation_count": 0,
"accepted_count": 0,
"adoption_sample_count": 1,
"activation_count": 1,
"accepted_count": 1,
"edited_count": 0,
"rejected_count": 0,
"missed_count": 0,
"failed_count": 0,
"adoption_rate": 0,
"adoption_rate": 100.0,
"missed_trigger_count": 0,
"wrong_trigger_count": 0,
"bad_output_count": 0,
@@ -1425,7 +1427,7 @@
"review_overdue_count": 0,
"risk_band": "low",
"event_types": {
"review_event": 1
"skill_activation": 1
},
"failure_types": {}
},
@@ -3122,7 +3124,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 33.11,
"duration_ms": 31.8,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3150,7 +3152,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.55,
"duration_ms": 31.63,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3173,7 +3175,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.73,
"duration_ms": 30.68,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3201,7 +3203,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.84,
"duration_ms": 32.69,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3224,7 +3226,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.53,
"duration_ms": 32.73,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3252,7 +3254,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.84,
"duration_ms": 34.03,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3275,7 +3277,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.54,
"duration_ms": 34.95,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3302,7 +3304,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.69,
"duration_ms": 34.6,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3325,7 +3327,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.54,
"duration_ms": 33.14,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3354,7 +3356,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 28.91,
"duration_ms": 36.79,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -3591,6 +3593,8 @@
"agreement_count": 0,
"disagreement_count": 0,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"pending_answer_hidden_count": 5,
"agreement_rate": null,
"needs_review": true,
"failure_count": 0
@@ -3609,7 +3613,8 @@
{
"case_id": "skill-package-contract",
"status": "pending",
"expected_winner_variant": "A",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -3618,7 +3623,8 @@
{
"case_id": "output-eval-expectation",
"status": "pending",
"expected_winner_variant": "A",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -3627,7 +3633,8 @@
{
"case_id": "ir-before-packaging",
"status": "pending",
"expected_winner_variant": "B",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -3636,7 +3643,8 @@
{
"case_id": "near-neighbor-boundary",
"status": "pending",
"expected_winner_variant": "B",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -3645,7 +3653,8 @@
{
"case_id": "file-backed-governed-package",
"status": "pending",
"expected_winner_variant": "B",
"expected_winner_variant": "",
"expected_revealed": false,
"reviewer_winner_variant": "",
"confidence": null,
"reason": "",
@@ -8215,7 +8224,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 153,
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
},
"failures": [],
"warnings": [],
@@ -10044,10 +10053,10 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 751,
"other_text_tokens": 910655,
"other_text_tokens": 911753,
"estimated_initial_load_tokens": 944,
"estimated_total_text_tokens": 911406,
"relevant_file_count": 388,
"estimated_total_text_tokens": 912504,
"relevant_file_count": 390,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 137.7
@@ -11043,8 +11052,8 @@
"report": "reports/adoption_drift_report.json",
"risk_band": "low",
"event_count": 1,
"adoption_sample_count": 0,
"adoption_rate": 0,
"adoption_sample_count": 1,
"adoption_rate": 100.0,
"candidate_count": 0
}
},
@@ -11875,7 +11884,7 @@
"adoption_drift": {
"ok": true,
"schema_version": "2.0",
"generated_at": "2026-06-13T12:24:01Z",
"generated_at": "2026-06-13T12:26:25Z",
"skill_dir": ".",
"privacy_contract": {
"storage": "local-first",
@@ -11899,14 +11908,14 @@
},
"summary": {
"event_count": 1,
"adoption_sample_count": 0,
"activation_count": 0,
"accepted_count": 0,
"adoption_sample_count": 1,
"activation_count": 1,
"accepted_count": 1,
"edited_count": 0,
"rejected_count": 0,
"missed_count": 0,
"failed_count": 0,
"adoption_rate": 0,
"adoption_rate": 100.0,
"missed_trigger_count": 0,
"wrong_trigger_count": 0,
"bad_output_count": 0,
@@ -11915,7 +11924,7 @@
"review_overdue_count": 0,
"risk_band": "low",
"event_types": {
"review_event": 1
"skill_activation": 1
},
"failure_types": {}
},
@@ -11923,29 +11932,29 @@
{
"skill": "yao-meta-skill",
"events": 1,
"adoption_events": 0,
"accepted": 0,
"adoption_events": 1,
"accepted": 1,
"edited": 0,
"rejected": 0,
"missed": 0,
"adoption_rate": 0
"adoption_rate": 100.0
}
],
"next_iteration_candidates": [],
"recent_events": [
{
"event": "review_event",
"event": "skill_activation",
"skill": "yao-meta-skill",
"version": "1.1.0",
"activation_type": "manual",
"outcome": "reviewed",
"activation_type": "explicit",
"outcome": "accepted",
"failure_type": "none",
"timestamp": "2026-06-13T12:00:00Z"
"timestamp": "2026-06-13T10:00:00Z"
}
],
"failures": [],
"artifacts": {
"events_jsonl": "reports/telemetry_events.jsonl",
"events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl",
"json": "reports/adoption_drift_report.json",
"markdown": "reports/adoption_drift_report.md"
}
@@ -11994,7 +12003,7 @@
"schema_version": "1.0",
"ok": true,
"skill_dir": ".",
"source": "reports/review_annotations_input.json",
"source": "tests/tmp_review_studio/empty_review_annotations_input.json",
"summary": {
"annotation_count": 0,
"open_count": 0,
@@ -12037,8 +12046,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb",
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
"compatibility": {
"openai": "pass",
@@ -12069,7 +12078,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -12094,7 +12103,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
}
]
},
@@ -12117,7 +12126,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"archive_entry_count": 494,
"failure_count": 0,
"warning_count": 0
@@ -12976,12 +12985,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
}
]
},
reports/runtime_permission_probes.json
+5 -5
View File
@@ -2,7 +2,7 @@
"schema_version": "1.0",
"ok": true,
"skill_dir": ".",
"package_dir": "tests/tmp_review_studio/dist",
"package_dir": "dist",
"expected_capabilities": [
"file_write",
"network",
@@ -22,7 +22,7 @@
{
"target": "openai",
"status": "pass",
"adapter": "tests/tmp_review_studio/dist/targets/openai/adapter.json",
"adapter": "dist/targets/openai/adapter.json",
"permission_model": "metadata-only",
"native_enforcement": false,
"metadata_fallback_explicit": true,
@@ -107,7 +107,7 @@
{
"target": "claude",
"status": "pass",
"adapter": "tests/tmp_review_studio/dist/targets/claude/adapter.json",
"adapter": "dist/targets/claude/adapter.json",
"permission_model": "neutral-source-plus-adapter",
"native_enforcement": false,
"metadata_fallback_explicit": true,
@@ -177,7 +177,7 @@
{
"target": "generic",
"status": "pass",
"adapter": "tests/tmp_review_studio/dist/targets/generic/adapter.json",
"adapter": "dist/targets/generic/adapter.json",
"permission_model": "agent-skills-compatible-metadata",
"native_enforcement": false,
"metadata_fallback_explicit": true,
@@ -247,7 +247,7 @@
{
"target": "vscode",
"status": "pass",
"adapter": "tests/tmp_review_studio/dist/targets/vscode/adapter.json",
"adapter": "dist/targets/vscode/adapter.json",
"permission_model": "vscode-workspace-trust-plus-metadata",
"native_enforcement": false,
"metadata_fallback_explicit": true,
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 153,
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `153`
- Package SHA256: `ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb`
- Package SHA256: `3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131`
## Failures
reports/skill-os-2-review.md
+3 -2
View File
@@ -29,9 +29,10 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
- Review Annotations v0 for reviewer comments tied to Review Studio gates, source/report paths, and optional line numbers; open blocker annotations now block the Review Studio decision.
- Review Studio now avoids over-claiming release readiness when blind A/B adjudication is still pending: the root Meta Skill is in `review` with score `92`, no blockers, two warnings, and explicit actions for Output Lab reviewer adjudication plus waiver handling.
- Review Studio Output Lab actions now link directly to `reports/output_review_decisions.json`, so pending blind A/B reviewer decisions have a concrete template instead of only a general adjudication warning.
- Output Review Adjudication now preserves blind-review integrity by hiding expected winners for pending or invalid reviewer decisions; answer keys are revealed only after a valid A/B decision exists for that case.
- Provider Output Eval Runner v0 so `python3 scripts/yao.py output-exec --provider-runner openai` can collect real provider-backed model evidence through a reviewed OpenAI Responses API compatible runner instead of ad hoc shell glue.
This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, a provider-backed output runner exists, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, real client telemetry capture, installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open.
This is still not the final world-class state. Target-native behavior contracts are now explicit, VS Code / Copilot package metadata is auditable, local output-eval command execution is wired, blind-review answers remain hidden until valid decisions exist, a provider-backed output runner exists, and Skill Atlas now consumes aggregate drift reports. Review Studio keeps pending human adjudication visible as a warning instead of treating it as a clean pass. Deeper provider-native execution transforms, real client telemetry capture, installer integration, real provider holdout runs, real human adjudication decisions, and native runtime permission enforcement remain open.
## Coverage Matrix
@@ -95,7 +96,7 @@ Next move: add real client or installer permission enforcement integration.
| Gate | Current Result |
| --- | --- |
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, and `0 / 5` reviewer decisions pending |
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden |
| Runtime Conformance | `5 / 5` targets passing |
| Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts |
| Trust | `0` secret findings, `1` pinned dependency file, `9` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `60 / 60` CLI help smoke checks passing across `69` scripts, source-contract hash scope explicit |
reports/skill-overview.html
+4 -4
View File
@@ -609,7 +609,7 @@
<ul class="list"><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">Skill 作者、复用团队和后续 reviewer。</span><span data-lang="en">Skill 作者、复用团队和后续 reviewer。</span></li><li><span data-lang="zh-CN">创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。</span><span data-lang="en">创建完成后建议先打开 reports/skill-overview.html，再继续扩展包体。</span></li></ul>
</aside>
</div>
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14610 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14610 个词/字。</span></small></article></div>
<div class="score-strip" aria-label="报告关键指标"><article class='score-chip'><span>完整度</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></article><article class='score-chip'><span>触发清晰</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></article><article class='score-chip'><span>证据充分</span><strong>100</strong><i style='--score:100%'></i><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></article><article class='score-chip'><span>上下文成本</span><strong>42</strong><i style='--score:42%'></i><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14640 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14640 个词/字。</span></small></article></div>
</section>
<section id="overview">
@@ -641,10 +641,10 @@
<article class="panel metrics-note">
<h3><span data-lang="zh-CN">指标判读</span><span data-lang="en">Reading</span></h3>
<p><span data-lang="zh-CN">先看雷达图判断能力短板，再看下方每项分数的证据原因。分数不是装饰数字，必须和本地文件、reports 证据或证据不足提示对应。</span><span data-lang="en">Read the radar first for weak spots, then inspect each score with its evidence. Scores must map to local files, reports, or explicit evidence gaps.</span></p>
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14610 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14610 个词/字。</span></small></li></ol>
<ol class='metric-summary-list'><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>完整度</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>触发清晰</b><em>100</em><small><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>证据充分</b><em>100</em><small><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可维护性</b><em>100</em><small><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">稳定</span><span data-lang="en">Stable</span></span><b>可迁移性</b><em>100</em><small><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></small></li><li><span class='metric-status'><span data-lang="zh-CN">关注</span><span data-lang="en">Watch</span></span><b>上下文成本</b><em>42</em><small><span data-lang="zh-CN">入口约 350 个词/字，references 约 14640 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14640 个词/字。</span></small></li></ol>
</article>
</div>
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在，便于人工阅读。</span><span data-lang="en">README.md 已存在，便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在，便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在，便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">compiled_targets.json 已存在。</span><span data-lang="en">compiled_targets.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制，可维护性较好。</span><span data-lang="en">入口文件保持克制，可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 350 个词/字，references 约 14610 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14610 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高，建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高，建议压缩入口或拆分 references。</span></li></ul></div></article></div>
<div class="metric-grid"><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>完整度</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 已存在，是 Skill 的入口。</span><span data-lang="en">SKILL.md 已存在，是 Skill 的入口。</span></li><li><span data-lang="zh-CN">README.md 已存在，便于人工阅读。</span><span data-lang="en">README.md 已存在，便于人工阅读。</span></li><li><span data-lang="zh-CN">agents/interface.yaml 已存在，便于跨平台适配。</span><span data-lang="en">agents/interface.yaml 已存在，便于跨平台适配。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>触发清晰</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">frontmatter description 已存在，具备基础路由面。</span><span data-lang="en">frontmatter description 已存在，具备基础路由面。</span></li><li><span data-lang="zh-CN">description 有足够长度说明任务边界。</span><span data-lang="en">description 有足够长度说明任务边界。</span></li><li><span data-lang="zh-CN">description 已包含使用场景或排除边界信号。</span><span data-lang="en">description 已包含使用场景或排除边界信号。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>证据充分</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">已生成 20 / 20 类报告证据。</span><span data-lang="en">已生成 20 / 20 类报告证据。</span></li><li><span data-lang="zh-CN">skill-ir.json 已存在。</span><span data-lang="en">skill-ir.json 已存在。</span></li><li><span data-lang="zh-CN">compiled_targets.json 已存在。</span><span data-lang="en">compiled_targets.json 已存在。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可维护性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">SKILL.md 约 350 个词/字。</span><span data-lang="en">SKILL.md 约 350 个词/字。</span></li><li><span data-lang="zh-CN">入口文件保持克制，可维护性较好。</span><span data-lang="en">入口文件保持克制，可维护性较好。</span></li><li><span data-lang="zh-CN">references/ 已承载扩展指导。</span><span data-lang="en">references/ 已承载扩展指导。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>可迁移性</span><strong>100</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">agents/interface.yaml 已存在。</span><span data-lang="en">agents/interface.yaml 已存在。</span></li><li><span data-lang="zh-CN">manifest.json 已存在。</span><span data-lang="en">manifest.json 已存在。</span></li><li><span data-lang="zh-CN">目标平台或 adapter target 已声明。</span><span data-lang="en">目标平台或 adapter target 已声明。</span></li></ul></div></article><article class='metric-card'><div class='metric-card-head'><span class='metric-label'>上下文成本</span><strong>42</strong></div><div class='metric-card-body'><ul class="compact-list"><li><span data-lang="zh-CN">入口约 350 个词/字，references 约 14640 个词/字。</span><span data-lang="en">入口约 350 个词/字，references 约 14640 个词/字。</span></li><li><span data-lang="zh-CN">分数越高代表上下文成本越低。</span><span data-lang="en">分数越高代表上下文成本越低。</span></li><li><span data-lang="zh-CN">上下文成本偏高，建议压缩入口或拆分 references。</span><span data-lang="en">上下文成本偏高，建议压缩入口或拆分 references。</span></li></ul></div></article></div>
</div>
</div>
</div>
@@ -720,7 +720,7 @@
<div>
<table>
<thead><tr><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th><th><span data-lang="zh-CN">证据</span><span data-lang="en">Evidence</span></th><th><span data-lang="zh-CN">建议</span><span data-lang="en">Action</span></th></tr></thead>
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简，并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span><span data-lang="en">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。</span><span data-lang="en">上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
<tbody><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">触发面保持精简，并锚定在 frontmatter description。</span><span data-lang="en">The trigger surface stays lean and anchored in the frontmatter description.</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span><span data-lang="en">已生成 Skill IR，核心语义可先于平台打包被审查和迁移。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">强项</span><span data-lang="en">Strength</span></td><td><span data-lang="zh-CN">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span><span data-lang="en">已生成目标编译报告，可审查 IR 到 OpenAI、Claude、generic 等目标契约的映射。</span></td><td><span data-lang="zh-CN">保留并复用</span><span data-lang="en">Keep</span></td></tr><tr><td><span data-lang="zh-CN">缺口</span><span data-lang="en">Gap</span></td><td><span data-lang="zh-CN">上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。</span><span data-lang="en">上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。</span></td><td><span data-lang="zh-CN">纳入下一轮修复</span><span data-lang="en">Fix next</span></td></tr></tbody>
</table>
<div class="two-col quality-panels">
<article class="panel">
reports/skill-overview.json
+16 -14
View File
@@ -96,7 +96,7 @@
"label": "上下文成本",
"score": 42,
"reasons": [
"入口约 350 个词/字，references 约 14610 个词/字。",
"入口约 350 个词/字，references 约 14640 个词/字。",
"分数越高代表上下文成本越低。",
"上下文成本偏高，建议压缩入口或拆分 references。"
]
@@ -200,7 +200,7 @@
"已生成 Output Review Adjudication，可记录盲评决策、一致率和待评审项。"
],
"gaps": [
"上下文成本需要补强：入口约 350 个词/字，references 约 14610 个词/字。"
"上下文成本需要补强：入口约 350 个词/字，references 约 14640 个词/字。"
],
"recommendations": [
"先改触发边界，再扩展工作流。",
@@ -871,6 +871,8 @@
"agreement_count": 0,
"disagreement_count": 0,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"pending_answer_hidden_count": 5,
"agreement_rate": null,
"needs_review": true,
"failure_count": 0
@@ -967,7 +969,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 153,
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
},
"skill_atlas": {
"skill_count": 12,
@@ -1005,8 +1007,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb",
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"package_sha256": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
"compatibility": {
"openai": "pass",
@@ -1037,7 +1039,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -1053,7 +1055,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20",
"archive_sha256": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6",
"archive_entry_count": 494,
"failure_count": 0,
"warning_count": 0
@@ -1128,12 +1130,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
}
]
},
@@ -1150,14 +1152,14 @@
"ok": true,
"summary": {
"event_count": 1,
"adoption_sample_count": 0,
"activation_count": 0,
"accepted_count": 0,
"adoption_sample_count": 1,
"activation_count": 1,
"accepted_count": 1,
"edited_count": 0,
"rejected_count": 0,
"missed_count": 0,
"failed_count": 0,
"adoption_rate": 0,
"adoption_rate": 100.0,
"missed_trigger_count": 0,
"wrong_trigger_count": 0,
"bad_output_count": 0,
@@ -1166,7 +1168,7 @@
"review_overdue_count": 0,
"risk_band": "low",
"event_types": {
"review_event": 1
"skill_activation": 1
},
"failure_types": {}
},
reports/upgrade_check.json
+4 -4
View File
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "23112632363494d9fcb52b0253bb23ffc9111e2dec2df0e9b06e3a49f304ff20"
"to": "a94b342c99204ed28626183e614c40bcdf7a1b8154fbb20256560258cd7910e6"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "ad55cf77f6b8faf7381b78372a93189c26e0e1833257bf7b508afd7ffdf0fbbb"
"to": "3eb57d2a31275058fdc10e5e26fb1c871dab2a2c72b532ddf4a990a530fdb131"
}
]
},
@@ -90,7 +90,7 @@
"artifacts": {
"previous_package": "registry/examples/yao-meta-skill-1.0.0.json",
"current_package": "reports/registry_audit.json",
"json": "tests/tmp_review_studio/upgrade_check.json",
"markdown": "tests/tmp_review_studio/upgrade_check.md"
"json": "reports/upgrade_check.json",
"markdown": "reports/upgrade_check.md"
}
}
scripts/adjudicate_output_review.py
+18 -5
View File
@@ -137,7 +137,8 @@ def adjudicate_pair(
{
"case_id": case_id,
"status": "pending",
"expected_winner_variant": expected,
"expected_winner_variant": "",
"expected_revealed": False,
"reviewer_winner_variant": "",
"confidence": None,
"reason": "",
@@ -156,7 +157,8 @@ def adjudicate_pair(
{
"case_id": case_id,
"status": "pending",
"expected_winner_variant": expected,
"expected_winner_variant": "",
"expected_revealed": False,
"reviewer_winner_variant": "",
"confidence": confidence,
"reason": reason,
@@ -164,7 +166,9 @@ def adjudicate_pair(
},
failures,
)
if reviewer not in {"A", "B"}:
if expected not in {"A", "B"}:
status = "invalid"
elif reviewer not in {"A", "B"}:
failures.append(f"{case_id}: winner_variant must be A or B")
status = "invalid"
elif confidence_failure:
@@ -172,11 +176,13 @@ def adjudicate_pair(
status = "invalid"
else:
status = "match" if reviewer == expected else "disagree"
expected_revealed = status in {"match", "disagree"}
return (
{
"case_id": case_id,
"status": status,
"expected_winner_variant": expected,
"expected_winner_variant": expected if expected_revealed else "",
"expected_revealed": expected_revealed,
"reviewer_winner_variant": reviewer,
"confidence": confidence,
"reason": reason,
@@ -193,6 +199,7 @@ def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str,
agreement_count = sum(1 for item in pairs if item["status"] == "match")
disagreement_count = sum(1 for item in pairs if item["status"] == "disagree")
invalid_decision_count = sum(1 for item in pairs if item["status"] == "invalid")
answer_revealed_count = sum(1 for item in pairs if item.get("expected_revealed"))
agreement_rate = round(agreement_count / judgment_count * 100, 2) if judgment_count else None
return {
"pair_count": pair_count,
@@ -201,6 +208,8 @@ def build_summary(pairs: list[dict[str, Any]], failures: list[str]) -> dict[str,
"agreement_count": agreement_count,
"disagreement_count": disagreement_count,
"invalid_decision_count": invalid_decision_count,
"answer_revealed_count": answer_revealed_count,
"pending_answer_hidden_count": sum(1 for item in pairs if item["status"] in {"pending", "invalid"} and not item.get("expected_revealed")),
"agreement_rate": agreement_rate,
"needs_review": pending_count > 0,
"failure_count": len(failures),
@@ -219,6 +228,8 @@ def render_markdown(payload: dict[str, Any]) -> str:
f"- Pending: `{summary['pending_count']}`",
f"- Agreement rate: `{summary['agreement_rate'] if summary['agreement_rate'] is not None else 'n/a'}`",
f"- Invalid decisions: `{summary['invalid_decision_count']}`",
f"- Answer keys revealed: `{summary['answer_revealed_count']}`",
f"- Pending/invalid answers hidden: `{summary['pending_answer_hidden_count']}`",
"",
]
if summary["judgment_count"] == 0:
@@ -227,6 +238,7 @@ def render_markdown(payload: dict[str, Any]) -> str:
"No reviewer decisions recorded yet.",
"",
"Generate a template with `--write-template`, fill `winner_variant` with `A` or `B`, then rerun adjudication.",
"Expected winners stay hidden until a valid reviewer decision is recorded.",
"",
]
)
@@ -241,9 +253,10 @@ def render_markdown(payload: dict[str, Any]) -> str:
for item in payload["pairs"]:
confidence = "" if item.get("confidence") is None else str(item["confidence"])
reason = str(item.get("reason", "")).replace("|", "\\|") or ""
expected = item.get("expected_winner_variant", "") if item.get("expected_revealed") else "hidden"
lines.append(
f"| {item['case_id']} | {item.get('reviewer_winner_variant', '') or 'pending'} | "
f"{item.get('expected_winner_variant', '') or 'missing'} | {item['status']} | {confidence} | {reason} |"
f"{expected} | {item['status']} | {confidence} | {reason} |"
)
if payload.get("failures"):
lines.extend(["", "## Failures", ""])
scripts/render_review_studio.py
+10 -1
View File
@@ -665,7 +665,16 @@ def render_html(report: dict[str, Any]) -> str:
)
review_panel = render_kv_grid(
output_review_summary,
["pair_count", "judgment_count", "pending_count", "agreement_count", "disagreement_count", "invalid_decision_count"],
[
"pair_count",
"judgment_count",
"pending_count",
"agreement_count",
"disagreement_count",
"invalid_decision_count",
"answer_revealed_count",
"pending_answer_hidden_count",
],
"review adjudication report missing",
)
conformance_panel = render_kv_grid(
scripts/review_studio_formatting.py
+2
View File
@@ -19,6 +19,7 @@ LABELS = {
"archive_entry_count": "Zip 条目",
"archive_present": "归档存在",
"archive_sha256": "归档哈希",
"answer_revealed_count": "答案揭示",
"baseline_pass_rate": "Baseline",
"breaking_change_count": "破坏变更",
"case_count": "案例数",
@@ -46,6 +47,7 @@ LABELS = {
"package_sha256": "包体哈希",
"pass_count": "通过数",
"pending_count": "待审",
"pending_answer_hidden_count": "答案隐藏",
"recorded_fixture_count": "记录样本",
"recommended_bump": "建议版本",
"residual_risk_count": "残余风险",
tests/verify_output_review_adjudication.py
+12
View File
@@ -75,7 +75,13 @@ def main() -> None:
assert pending_payload["summary"]["pending_count"] == 5, pending_payload
assert pending_payload["summary"]["agreement_rate"] is None, pending_payload
assert pending_payload["summary"]["needs_review"], pending_payload
assert pending_payload["summary"]["answer_revealed_count"] == 0, pending_payload
assert pending_payload["summary"]["pending_answer_hidden_count"] == 5, pending_payload
assert all(not item["expected_winner_variant"] and not item["expected_revealed"] for item in pending_payload["pairs"]), pending_payload
assert "No reviewer decisions recorded yet" in pending_md.read_text(encoding="utf-8"), pending_md
pending_text = pending_md.read_text(encoding="utf-8")
assert "| skill-package-contract | pending | hidden | pending |" in pending_text, pending_text
assert "| skill-package-contract | pending | A | pending |" not in pending_text, pending_text
template_path = tmp_root / "output_review_decisions.json"
template_proc = run(
@@ -139,7 +145,10 @@ def main() -> None:
assert filled_payload["summary"]["pending_count"] == 0, filled_payload
assert filled_payload["summary"]["agreement_count"] == 5, filled_payload
assert filled_payload["summary"]["agreement_rate"] == 100.0, filled_payload
assert filled_payload["summary"]["answer_revealed_count"] == 5, filled_payload
assert filled_payload["summary"]["pending_answer_hidden_count"] == 0, filled_payload
assert all(item["status"] == "match" for item in filled_payload["pairs"]), filled_payload
assert all(item["expected_winner_variant"] in {"A", "B"} and item["expected_revealed"] for item in filled_payload["pairs"]), filled_payload
invalid = {
"schema_version": "1.0",
@@ -176,6 +185,9 @@ def main() -> None:
invalid_payload = json.loads(invalid_proc.stdout)
assert not invalid_payload["ok"], invalid_payload
assert invalid_payload["summary"]["invalid_decision_count"] == 1, invalid_payload
assert invalid_payload["summary"]["answer_revealed_count"] == 0, invalid_payload
assert invalid_payload["pairs"][0]["expected_winner_variant"] == "", invalid_payload
assert invalid_payload["pairs"][0]["expected_revealed"] is False, invalid_payload
assert invalid_payload["failures"], invalid_payload
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
tests/verify_review_studio.py
+5
View File
@@ -300,6 +300,9 @@ def main() -> None:
assert full_payload["data"]["output_execution"]["summary"]["recorded_fixture_count"] == 0, full_payload["data"]["output_execution"]
assert full_payload["data"]["output_execution"]["summary"]["model_executed_count"] == 0, full_payload["data"]["output_execution"]
assert full_payload["data"]["output_review_adjudication"]["summary"]["pending_count"] == 5, full_payload["data"]["output_review_adjudication"]
assert full_payload["data"]["output_review_adjudication"]["summary"]["answer_revealed_count"] == 0, full_payload["data"]["output_review_adjudication"]
assert full_payload["data"]["output_review_adjudication"]["summary"]["pending_answer_hidden_count"] == 5, full_payload["data"]["output_review_adjudication"]
assert all(not item["expected_revealed"] for item in full_payload["data"]["output_review_adjudication"]["pairs"]), full_payload["data"]["output_review_adjudication"]
assert full_payload["data"]["review_annotations"]["summary"]["annotation_count"] == 0, full_payload["data"]["review_annotations"]
assert full_payload["data"]["compiled_targets"]["summary"]["target_count"] >= 4, full_payload["data"]["compiled_targets"]
assert full_payload["data"]["compiled_targets"]["summary"]["block_count"] == 0, full_payload["data"]["compiled_targets"]
@@ -355,6 +358,8 @@ def main() -> None:
assert "执行证据" in html, html
assert "盲评包" in html, html[:5000]
assert "审定报告" in html, html
assert "答案揭示" in html, html
assert "答案隐藏" in html, html
assert "注册审计" in html, html[:3000]
assert "包体验证" in html, html[:5000]
assert "Upgrade" in html, html[:6000]