feat: add output review cockpit
This commit is contained in:
yaojingang
+1
-1
@@ -16,7 +16,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
@@ -16,8 +16,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492",
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -48,7 +48,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-13T23:15:43Z",
|
||||
"generated_at": "2026-06-13T23:31:32Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao.py",
|
||||
"lines": 864,
|
||||
"lines": 867,
|
||||
"kind": "cli-script",
|
||||
"severity": "pass",
|
||||
"recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator."
|
||||
@@ -55,7 +55,7 @@
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao_cli_parser.py",
|
||||
"lines": 727,
|
||||
"lines": 728,
|
||||
"kind": "internal-module",
|
||||
"severity": "pass",
|
||||
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
|
||||
|
||||
@@ -27,10 +27,10 @@ No file-size hotspots found.
|
||||
| --- | ---: | --- | --- |
|
||||
| `scripts/render_review_studio.py` | `899` | `cli-script` | `pass` |
|
||||
| `tests/verify_yao_cli.py` | `898` | `test` | `pass` |
|
||||
| `scripts/yao.py` | `864` | `cli-script` | `pass` |
|
||||
| `scripts/yao.py` | `867` | `cli-script` | `pass` |
|
||||
| `scripts/skill_report_model.py` | `792` | `internal-module` | `pass` |
|
||||
| `scripts/compile_skill.py` | `734` | `cli-script` | `pass` |
|
||||
| `scripts/yao_cli_parser.py` | `727` | `internal-module` | `pass` |
|
||||
| `scripts/yao_cli_parser.py` | `728` | `internal-module` | `pass` |
|
||||
| `scripts/optimize_description.py` | `723` | `cli-script` | `pass` |
|
||||
| `scripts/trust_check.py` | `714` | `internal-module` | `pass` |
|
||||
| `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` |
|
||||
|
||||
@@ -3,14 +3,12 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"skill_dir": ".",
|
||||
"commit": "0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b",
|
||||
"commit": "834665b73c1422486be8cb1b1dd826fa58e81be7",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": true,
|
||||
"changed_file_count": 65,
|
||||
"changed_file_count": 58,
|
||||
"sample": [
|
||||
" M evidence/world_class/templates/native-permission-enforcement.intake.json",
|
||||
" M references/trust-security-method.md",
|
||||
" M registry/index.json",
|
||||
" M registry/packages/yao-meta-skill.json",
|
||||
" M reports/adoption_drift_report.json",
|
||||
@@ -20,7 +18,9 @@
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/compiled_targets.json",
|
||||
" M reports/context_budget.json"
|
||||
" M reports/context_budget.json",
|
||||
" M reports/install_simulation.json",
|
||||
" M reports/install_simulation.md"
|
||||
]
|
||||
},
|
||||
"summary": {
|
||||
@@ -42,7 +42,7 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 65
|
||||
"changed_file_count": 58
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -116,7 +116,7 @@
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7967,
|
||||
"sha256": "5c02c5db6c557ba4f595f0616e5792e61e7f54c51b3a32dddfab9f3a771f7795"
|
||||
"sha256": "94be89b0096ba1890e95f7ecfe2a25f603f507aca699843af5c466a1e6812848"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
@@ -151,7 +151,7 @@
|
||||
"path": "reports/security_trust_report.json",
|
||||
"exists": true,
|
||||
"bytes": 95873,
|
||||
"sha256": "bce52b7b783cf2caa79c470ae6733073a67870703d0b33b47ad3251264196ef7"
|
||||
"sha256": "e5bbb8250aeeee8b77cac94c7308484d27c90cf7bc241aea4e66791ec565f83c"
|
||||
},
|
||||
{
|
||||
"label": "python_compatibility",
|
||||
@@ -165,21 +165,21 @@
|
||||
"path": "reports/registry_audit.json",
|
||||
"exists": true,
|
||||
"bytes": 3183,
|
||||
"sha256": "03edb72bdd407f9ee8ae7c37ce15c03ce115ee7c621d4de446cae6ce56a08705"
|
||||
"sha256": "3ea8d598452cf8995d403164bf3d204b2cc40b07accb8ea4883486e805c74e31"
|
||||
},
|
||||
{
|
||||
"label": "package_verification",
|
||||
"path": "reports/package_verification.json",
|
||||
"exists": true,
|
||||
"bytes": 19325,
|
||||
"sha256": "212ddb0a9c5a2becc3bbc64b209d23c018123557cebbfccb78faed08c5572184"
|
||||
"sha256": "aef6143865fb67bcf2bc9f75f2cba8308deb2498180257c585050b3b46fb1534"
|
||||
},
|
||||
{
|
||||
"label": "install_simulation",
|
||||
"path": "reports/install_simulation.json",
|
||||
"exists": true,
|
||||
"bytes": 8604,
|
||||
"sha256": "69d98943271497a0c497e93a92352c76cda61c77876f9893efd80fdb99383f1e"
|
||||
"sha256": "eebfba42d71e4082d3c801fd126e59c6401b3f8c00a00625c70eb82f927ebd65"
|
||||
},
|
||||
{
|
||||
"label": "skill_os2_audit",
|
||||
@@ -213,8 +213,8 @@
|
||||
"label": "world_class_claim_guard",
|
||||
"path": "reports/world_class_claim_guard.json",
|
||||
"exists": true,
|
||||
"bytes": 8179,
|
||||
"sha256": "299a88eb7b47d16761d66980e01ccd8164609e344c3c98bb0b8a67b56cf68edd"
|
||||
"bytes": 8267,
|
||||
"sha256": "79c5b29f0ff639189aa65f3487afab13150e95c7fa7a6a5b59309b1e87873c25"
|
||||
}
|
||||
],
|
||||
"missing_artifacts": [],
|
||||
|
||||
@@ -1,7 +1,7 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-14`
|
||||
Commit: `0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b`
|
||||
Commit: `834665b73c1422486be8cb1b1dd826fa58e81be7`
|
||||
Working tree dirty at generation: `true`
|
||||
|
||||
## Summary
|
||||
@@ -16,7 +16,7 @@ Working tree dirty at generation: `true`
|
||||
- provider evidence complete: `false`
|
||||
- human review complete: `false`
|
||||
- world-class ready: `false`
|
||||
- changed files at generation: `65`
|
||||
- changed files at generation: `58`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.
|
||||
|
||||
@@ -40,21 +40,21 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `5c02c5db6c55` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `94be89b0096b` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `bce52b7b783c` |
|
||||
| trust_report | `reports/security_trust_report.json` | present | `e5bbb8250aee` |
|
||||
| python_compatibility | `reports/python_compatibility.json` | present | `e5dd973d0b5c` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `03edb72bdd40` |
|
||||
| package_verification | `reports/package_verification.json` | present | `212ddb0a9c5a` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `69d989432714` |
|
||||
| registry_audit | `reports/registry_audit.json` | present | `3ea8d598452c` |
|
||||
| package_verification | `reports/package_verification.json` | present | `aef6143865fb` |
|
||||
| install_simulation | `reports/install_simulation.json` | present | `eebfba42d71e` |
|
||||
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `a1ab5e625fbc` |
|
||||
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `02e49b66159d` |
|
||||
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0b52d0d1528b` |
|
||||
| world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `5fbfcd35ac6a` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `299a88eb7b47` |
|
||||
| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `79c5b29f0ff6` |
|
||||
|
||||
## Reproduction Commands
|
||||
|
||||
|
||||
@@ -6,10 +6,10 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 751,
|
||||
"other_text_tokens": 1151542,
|
||||
"other_text_tokens": 1154440,
|
||||
"estimated_initial_load_tokens": 944,
|
||||
"estimated_total_text_tokens": 1152293,
|
||||
"relevant_file_count": 490,
|
||||
"estimated_total_text_tokens": 1155191,
|
||||
"relevant_file_count": 491,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
"quality_density": 137.7
|
||||
|
||||
@@ -8,7 +8,7 @@
|
||||
"installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 567,
|
||||
"archive_entry_count": 574,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.82,
|
||||
"duration_ms": 28.21,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.2,
|
||||
"duration_ms": 27.95,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.79,
|
||||
"duration_ms": 28.36,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.16,
|
||||
"duration_ms": 27.76,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.57,
|
||||
"duration_ms": 28.5,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.12,
|
||||
"duration_ms": 28.05,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.61,
|
||||
"duration_ms": 28.23,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.9,
|
||||
"duration_ms": 28.26,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.06,
|
||||
"duration_ms": 28.66,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.98,
|
||||
"duration_ms": 28.46,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 27.82 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 28.2 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.79 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.16 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.57 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.12 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.61 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.9 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.06 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.98 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 28.21 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.95 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 28.36 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 27.76 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.5 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.05 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.23 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.26 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.66 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.46 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -0,0 +1,220 @@
|
||||
<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Output Review Kit</title>
|
||||
<style>
|
||||
:root { --ink:#1B365D; --text:#202124; --muted:#6f6a63; --line:#e8e1d8; --soft:#f8f6f2; --warn:#9b4d0f; --ok:#1f6f43; }
|
||||
* { box-sizing:border-box; }
|
||||
body { margin:0; background:#fff; color:var(--text); font:16px/1.55 Georgia, "Times New Roman", serif; }
|
||||
.shell { max-width:1180px; margin:0 auto; padding:36px 24px 72px; }
|
||||
.topbar { position:sticky; top:0; z-index:10; background:rgba(255,255,255,.96); border-bottom:1px solid var(--line); backdrop-filter:blur(8px); }
|
||||
.topbar-inner { max-width:1180px; margin:0 auto; padding:12px 24px; display:flex; justify-content:space-between; gap:18px; align-items:center; }
|
||||
.brand { color:var(--ink); font-weight:700; letter-spacing:0; }
|
||||
.links { display:flex; gap:12px; flex-wrap:wrap; }
|
||||
.links a { color:var(--ink); text-decoration:none; border-bottom:1px solid transparent; }
|
||||
.links a:hover { border-color:var(--ink); }
|
||||
.hero { border-bottom:1px solid var(--line); padding:34px 0 28px; }
|
||||
.eyebrow { color:var(--ink); text-transform:uppercase; font-size:12px; letter-spacing:0; font-weight:700; }
|
||||
h1 { margin:8px 0 12px; color:var(--ink); font-size:58px; line-height:1.02; letter-spacing:0; }
|
||||
.lede { max-width:760px; color:var(--muted); font-size:20px; }
|
||||
.stats { display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin:28px 0; }
|
||||
.stats div, .panel, .case-card { border:1px solid var(--line); border-radius:8px; background:#fff; }
|
||||
.stats div { padding:16px; }
|
||||
.stats span { display:block; color:var(--muted); font-size:13px; }
|
||||
.stats strong { color:var(--ink); font-size:34px; line-height:1; }
|
||||
.grid { display:grid; grid-template-columns:minmax(0,1fr) minmax(280px,.42fr); gap:18px; align-items:start; }
|
||||
.panel { padding:20px; }
|
||||
h2, h3, h4 { color:var(--ink); letter-spacing:0; }
|
||||
h2 { margin:0 0 14px; font-size:28px; }
|
||||
h3 { margin:0; font-size:24px; }
|
||||
h4 { margin:0 0 10px; font-size:16px; }
|
||||
ol, ul { padding-left:22px; }
|
||||
code, pre { font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }
|
||||
pre { white-space:pre-wrap; overflow-wrap:anywhere; margin:0; color:#2b2b2b; font-size:13px; line-height:1.5; }
|
||||
.case-card { margin:22px 0; padding:22px; scroll-margin-top:72px; }
|
||||
.case-head, .case-foot { display:flex; gap:14px; justify-content:space-between; align-items:flex-start; flex-wrap:wrap; }
|
||||
.case-index { color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:0; }
|
||||
.status-pill { border:1px solid var(--line); border-radius:999px; padding:4px 10px; color:var(--ink); background:var(--soft); font-size:13px; }
|
||||
.status-pill.ready-for-adjudication { color:var(--ok); }
|
||||
.status-pill.needs-fix { color:var(--warn); }
|
||||
.prompt { color:var(--muted); font-size:18px; }
|
||||
.rubric { margin:18px 0; padding:16px; background:var(--soft); border-radius:8px; }
|
||||
.rubric ul { list-style:none; padding:0; margin:0; display:grid; gap:8px; }
|
||||
.rubric li { display:grid; grid-template-columns:160px 1fr 44px; gap:12px; align-items:start; border-top:1px solid var(--line); padding-top:8px; }
|
||||
.rubric li:first-child { border-top:0; padding-top:0; }
|
||||
.rubric span { color:var(--ink); font-weight:700; overflow-wrap:anywhere; }
|
||||
.rubric p { margin:0; }
|
||||
.rubric small { text-align:right; color:var(--muted); }
|
||||
.variants { display:grid; grid-template-columns:1fr 1fr; gap:16px; }
|
||||
.variant { border:1px solid var(--line); border-radius:8px; padding:16px; min-width:0; }
|
||||
.case-foot { margin-top:16px; padding-top:14px; border-top:1px solid var(--line); color:var(--muted); font-size:13px; }
|
||||
.case-foot strong { flex-basis:100%; color:var(--text); font-weight:400; }
|
||||
.template { background:#101820; color:#f7f2e8; border-radius:8px; padding:16px; }
|
||||
@media (max-width:820px) { .stats, .grid, .variants { grid-template-columns:1fr; } .rubric li { grid-template-columns:1fr; } .rubric small { text-align:left; } h1 { font-size:38px; } }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="topbar"><div class="topbar-inner"><span class="brand">Output Review Kit</span><div class="links"><a href="#flow">Flow</a><a href="#cases">Cases</a><a href="#template">Template</a></div></div></nav>
|
||||
<main class="shell">
|
||||
<section class="hero">
|
||||
<span class="eyebrow">Blind A/B Human Review</span>
|
||||
<h1>Reviewer cockpit for output quality decisions</h1>
|
||||
<p class="lede">Compare visible Variant A and Variant B outputs, fill the decision file, then run adjudication. The answer key is intentionally hidden from this page.</p>
|
||||
<div class="stats"><div><span>Cases</span><strong>5</strong></div><div><span>Ready</span><strong></strong></div><div><span>Pending</span><strong>5</strong></div><div><span>Invalid</span><strong></strong></div></div>
|
||||
</section>
|
||||
<section class="grid" id="flow">
|
||||
<article class="panel"><h2>Review Flow</h2><ol><li>Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case.</li><li>Record choices in reports/output_review_decisions.json without opening the answer key.</li><li>Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.</li><li>Run python3 scripts/yao.py output-review after choices are recorded.</li><li>Refresh python3 scripts/yao.py review-studio . before asking for release approval.</li></ol></article>
|
||||
<aside class="panel"><h2>Privacy</h2><ul><li>The answer key is intentionally withheld from this kit.</li><li>Do not inspect hidden labels or expected winners before decisions are recorded.</li><li>Do not paste private user data into decision reasons.</li><li>Pending decisions must stay pending instead of being counted as human agreement.</li></ul></aside>
|
||||
</section>
|
||||
<section id="cases">
|
||||
<article class="case-card" id="case-skill-package-contract">
|
||||
<header class="case-head">
|
||||
<div>
|
||||
<span class="case-index">Case 01</span>
|
||||
<h3>skill-package-contract</h3>
|
||||
</div>
|
||||
<span class="status-pill awaiting-decision">Awaiting</span>
|
||||
</header>
|
||||
<p class="prompt">Turn this repeated workflow into a reusable team skill package.</p>
|
||||
<section class="rubric"><h4>Rubric</h4><ul><li><span>has-entrypoint</span><p>Output names the routeable Skill entrypoint.</p><small>1.0</small></li><li><span>has-interface</span><p>Output includes neutral interface metadata.</p><small>1.0</small></li><li><span>has-report</span><p>Output points reviewer to generated overview report.</p><small>1.0</small></li><li><span>has-resource-boundary</span><p>Output preserves entrypoint/resource separation.</p><small>1.0</small></li></ul></section>
|
||||
<section class="variants" aria-label="Blind output variants">
|
||||
<div class="variant"><h4>Variant A</h4><pre>Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/.</pre></div>
|
||||
<div class="variant"><h4>Variant B</h4><pre>I can write a prompt for that workflow and include a short checklist.</pre></div>
|
||||
</section>
|
||||
<footer class="case-foot">
|
||||
<span>Winner recorded: false</span>
|
||||
<span>Confidence: false</span>
|
||||
<span>Reason: false</span>
|
||||
<strong>Decision template exists but this row is still blank.</strong>
|
||||
</footer>
|
||||
</article>
|
||||
|
||||
<article class="case-card" id="case-output-eval-expectation">
|
||||
<header class="case-head">
|
||||
<div>
|
||||
<span class="case-index">Case 02</span>
|
||||
<h3>output-eval-expectation</h3>
|
||||
</div>
|
||||
<span class="status-pill awaiting-decision">Awaiting</span>
|
||||
</header>
|
||||
<p class="prompt">Upgrade this production skill so we know whether the generated output is better than baseline.</p>
|
||||
<section class="rubric"><h4>Rubric</h4><ul><li><span>has-baseline-comparison</span><p>Output explicitly compares with-skill and baseline outputs.</p><small>1.0</small></li><li><span>has-assertion-grading</span><p>Output includes objective assertion grading.</p><small>1.0</small></li><li><span>has-scorecard</span><p>Output produces a scorecard report path.</p><small>1.0</small></li><li><span>has-failure-taxonomy</span><p>Output records failure taxonomy.</p><small>1.0</small></li></ul></section>
|
||||
<section class="variants" aria-label="Blind output variants">
|
||||
<div class="variant"><h4>Variant A</h4><pre>Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md.</pre></div>
|
||||
<div class="variant"><h4>Variant B</h4><pre>Add more examples and run the trigger eval again.</pre></div>
|
||||
</section>
|
||||
<footer class="case-foot">
|
||||
<span>Winner recorded: false</span>
|
||||
<span>Confidence: false</span>
|
||||
<span>Reason: false</span>
|
||||
<strong>Decision template exists but this row is still blank.</strong>
|
||||
</footer>
|
||||
</article>
|
||||
|
||||
<article class="case-card" id="case-ir-before-packaging">
|
||||
<header class="case-head">
|
||||
<div>
|
||||
<span class="case-index">Case 03</span>
|
||||
<h3>ir-before-packaging</h3>
|
||||
</div>
|
||||
<span class="status-pill awaiting-decision">Awaiting</span>
|
||||
</header>
|
||||
<p class="prompt">Package this skill for OpenAI, Claude, Agent Skills, and generic targets.</p>
|
||||
<section class="rubric"><h4>Rubric</h4><ul><li><span>has-ir-first</span><p>Output requires Skill IR before packaging.</p><small>1.0</small></li><li><span>has-core-ir-fields</span><p>Output lists core IR contract fields.</p><small>1.0</small></li><li><span>has-targets</span><p>Output names requested runtime targets.</p><small>1.0</small></li><li><span>has-semantic-preservation</span><p>Output says targets must preserve the capability contract.</p><small>1.0</small></li></ul></section>
|
||||
<section class="variants" aria-label="Blind output variants">
|
||||
<div class="variant"><h4>Variant A</h4><pre>Export folders for each platform and adjust files as needed.</pre></div>
|
||||
<div class="variant"><h4>Variant B</h4><pre>Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract.</pre></div>
|
||||
</section>
|
||||
<footer class="case-foot">
|
||||
<span>Winner recorded: false</span>
|
||||
<span>Confidence: false</span>
|
||||
<span>Reason: false</span>
|
||||
<strong>Decision template exists but this row is still blank.</strong>
|
||||
</footer>
|
||||
</article>
|
||||
|
||||
<article class="case-card" id="case-near-neighbor-boundary">
|
||||
<header class="case-head">
|
||||
<div>
|
||||
<span class="case-index">Case 04</span>
|
||||
<h3>near-neighbor-boundary</h3>
|
||||
</div>
|
||||
<span class="status-pill awaiting-decision">Awaiting</span>
|
||||
</header>
|
||||
<p class="prompt">I only need a one-off summary of these notes, not a reusable process.</p>
|
||||
<section class="rubric"><h4>Rubric</h4><ul><li><span>declines-skill</span><p>Output refuses unnecessary skill creation for one-off work.</p><small>1.0</small></li><li><span>names-near-neighbor</span><p>Output labels the request as near-neighbor instead of owned work.</p><small>1.0</small></li><li><span>requires-reuse-signal</span><p>Output asks for repeat-use evidence before packaging.</p><small>1.0</small></li></ul></section>
|
||||
<section class="variants" aria-label="Blind output variants">
|
||||
<div class="variant"><h4>Variant A</h4><pre>Create a SKILL.md and a reusable workflow anyway so future notes can use it.</pre></div>
|
||||
<div class="variant"><h4>Variant B</h4><pre>Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract.</pre></div>
|
||||
</section>
|
||||
<footer class="case-foot">
|
||||
<span>Winner recorded: false</span>
|
||||
<span>Confidence: false</span>
|
||||
<span>Reason: false</span>
|
||||
<strong>Decision template exists but this row is still blank.</strong>
|
||||
</footer>
|
||||
</article>
|
||||
|
||||
<article class="case-card" id="case-file-backed-governed-package">
|
||||
<header class="case-head">
|
||||
<div>
|
||||
<span class="case-index">Case 05</span>
|
||||
<h3>file-backed-governed-package</h3>
|
||||
</div>
|
||||
<span class="status-pill awaiting-decision">Awaiting</span>
|
||||
</header>
|
||||
<p class="prompt">Turn the attached release brief source into a governed skill package.</p>
|
||||
<section class="rubric"><h4>Rubric</h4><ul><li><span>uses-file-backed-evidence</span><p>Output names file-backed source evidence.</p><small>1.0</small></li><li><span>has-governance</span><p>Output preserves governed ownership metadata.</p><small>1.0</small></li><li><span>has-output-contract</span><p>Output preserves output and rollback boundaries.</p><small>1.0</small></li><li><span>has-trust-and-scorecard</span><p>Output requires trust and output scorecard artifacts.</p><small>1.0</small></li><li><span>does-not-invent-evidence</span><p>Output forbids invented launch evidence.</p><small>1.0</small></li></ul></section>
|
||||
<section class="variants" aria-label="Blind output variants">
|
||||
<div class="variant"><h4>Variant A</h4><pre>Draft a release announcement with the changelog, support notes, and owner name.</pre></div>
|
||||
<div class="variant"><h4>Variant B</h4><pre>Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them.</pre></div>
|
||||
</section>
|
||||
<footer class="case-foot">
|
||||
<span>Winner recorded: false</span>
|
||||
<span>Confidence: false</span>
|
||||
<span>Reason: false</span>
|
||||
<strong>Decision template exists but this row is still blank.</strong>
|
||||
</footer>
|
||||
</article></section>
|
||||
<section class="panel" id="template"><h2>Decision Template</h2><p>Use this shape in reports/output_review_decisions.json; leave a case blank when the reviewer is not ready.</p><pre class="template">{
|
||||
"schema_version": "1.0",
|
||||
"reviewer": "",
|
||||
"reviewed_at": "",
|
||||
"decisions": [
|
||||
{
|
||||
"case_id": "skill-package-contract",
|
||||
"winner_variant": "",
|
||||
"confidence": null,
|
||||
"reason": ""
|
||||
},
|
||||
{
|
||||
"case_id": "output-eval-expectation",
|
||||
"winner_variant": "",
|
||||
"confidence": null,
|
||||
"reason": ""
|
||||
},
|
||||
{
|
||||
"case_id": "ir-before-packaging",
|
||||
"winner_variant": "",
|
||||
"confidence": null,
|
||||
"reason": ""
|
||||
},
|
||||
{
|
||||
"case_id": "near-neighbor-boundary",
|
||||
"winner_variant": "",
|
||||
"confidence": null,
|
||||
"reason": ""
|
||||
},
|
||||
{
|
||||
"case_id": "file-backed-governed-package",
|
||||
"winner_variant": "",
|
||||
"confidence": null,
|
||||
"reason": ""
|
||||
}
|
||||
]
|
||||
}</pre></section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
@@ -15,6 +15,7 @@
|
||||
"artifacts": {
|
||||
"reviewer_kit_json": "reports/output_review_kit.json",
|
||||
"reviewer_kit_markdown": "reports/output_review_kit.md",
|
||||
"reviewer_kit_html": "reports/output_review_kit.html",
|
||||
"blind_pack_json": "reports/output_blind_review_pack.json",
|
||||
"blind_pack_markdown": "reports/output_blind_review_pack.md",
|
||||
"decisions": "reports/output_review_decisions.json",
|
||||
|
||||
@@ -8,8 +8,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_entry_count": 567,
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"archive_entry_count": 568,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
|
||||
@@ -4,7 +4,7 @@
|
||||
- Package directory: `dist`
|
||||
- Targets: `4 / 4` adapters present
|
||||
- Archive present: `True`
|
||||
- Archive SHA256: `2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34`
|
||||
- Archive SHA256: `fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7`
|
||||
- Failures: `0`
|
||||
- Warnings: `0`
|
||||
|
||||
|
||||
@@ -21,8 +21,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492",
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -53,7 +53,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -78,7 +78,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,8 +6,8 @@
|
||||
- Maturity: `governed`
|
||||
- Owner: `Yao Team`
|
||||
- License: `MIT`
|
||||
- Package SHA256: `09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492`
|
||||
- Archive SHA256: `2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34`
|
||||
- Package SHA256: `5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045`
|
||||
- Archive SHA256: `fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7`
|
||||
- Install simulated: `True`
|
||||
|
||||
## Compatibility
|
||||
|
||||
+12
-12
File diff suppressed because one or more lines are too long
+83
-76
@@ -107,7 +107,7 @@
|
||||
"key": "operations-loop",
|
||||
"label": "运营回路",
|
||||
"status": "pass",
|
||||
"detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low",
|
||||
"detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low",
|
||||
"evidence": "reports/adoption_drift_report.json",
|
||||
"link": "adoption_drift_report.md"
|
||||
},
|
||||
@@ -179,7 +179,7 @@
|
||||
"priority": "warning",
|
||||
"summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。",
|
||||
"why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。",
|
||||
"source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md",
|
||||
"source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md",
|
||||
"source_refs": [
|
||||
{
|
||||
"path": "evals/output/cases.jsonl",
|
||||
@@ -214,12 +214,12 @@
|
||||
"link": "output_blind_review_pack.md"
|
||||
},
|
||||
{
|
||||
"path": "reports/output_review_decisions.json",
|
||||
"label": "review decisions template",
|
||||
"path": "reports/output_review_kit.html",
|
||||
"label": "reviewer cockpit",
|
||||
"kind": "report",
|
||||
"line": 5,
|
||||
"line": 6,
|
||||
"exists": true,
|
||||
"link": "output_review_decisions.json"
|
||||
"link": "output_review_kit.html"
|
||||
},
|
||||
{
|
||||
"path": "reports/output_review_adjudication.md",
|
||||
@@ -382,6 +382,7 @@
|
||||
"output_execution": "reports/output_execution_runs.md",
|
||||
"output_blind_review": "reports/output_blind_review_pack.md",
|
||||
"output_review_kit": "reports/output_review_kit.md",
|
||||
"output_review_kit_html": "reports/output_review_kit.html",
|
||||
"output_review_decisions": "reports/output_review_decisions.json",
|
||||
"output_review_adjudication": "reports/output_review_adjudication.md",
|
||||
"benchmark_reproducibility": "reports/benchmark_reproducibility.md",
|
||||
@@ -860,10 +861,10 @@
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 199
|
||||
"file_count": 200
|
||||
}
|
||||
],
|
||||
"file_count": 356,
|
||||
"file_count": 357,
|
||||
"folder_count": 4,
|
||||
"distribution": [
|
||||
{
|
||||
@@ -896,7 +897,7 @@
|
||||
},
|
||||
{
|
||||
"label": "reports",
|
||||
"value": 199
|
||||
"value": 200
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1029,7 +1030,7 @@
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 199
|
||||
"file_count": 200
|
||||
}
|
||||
],
|
||||
"strengths": [
|
||||
@@ -1293,6 +1294,7 @@
|
||||
"artifacts": {
|
||||
"reviewer_kit_json": "reports/output_review_kit.json",
|
||||
"reviewer_kit_markdown": "reports/output_review_kit.md",
|
||||
"reviewer_kit_html": "reports/output_review_kit.html",
|
||||
"blind_pack_json": "reports/output_blind_review_pack.json",
|
||||
"blind_pack_markdown": "reports/output_blind_review_pack.md",
|
||||
"decisions": "reports/output_review_decisions.json",
|
||||
@@ -1345,9 +1347,9 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 63
|
||||
"changed_file_count": 33
|
||||
},
|
||||
"commit": "0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b",
|
||||
"commit": "834665b73c1422486be8cb1b1dd826fa58e81be7",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
@@ -1451,7 +1453,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 177,
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1489,8 +1491,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492",
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1521,7 +1523,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1537,8 +1539,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_entry_count": 567,
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"archive_entry_count": 568,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -1549,7 +1551,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 567,
|
||||
"archive_entry_count": 568,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -1616,12 +1618,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1638,14 +1640,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1654,7 +1656,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1873,7 +1875,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -2112,7 +2114,7 @@
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"provenance_requirements": [
|
||||
"real external client source",
|
||||
@@ -2140,7 +2142,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -3790,7 +3792,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.46,
|
||||
"duration_ms": 25.88,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3818,7 +3820,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 26.0,
|
||||
"duration_ms": 25.25,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3841,7 +3843,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.44,
|
||||
"duration_ms": 25.73,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3869,7 +3871,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.47,
|
||||
"duration_ms": 26.83,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3892,7 +3894,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.76,
|
||||
"duration_ms": 28.58,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3920,7 +3922,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.7,
|
||||
"duration_ms": 27.09,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3943,7 +3945,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.55,
|
||||
"duration_ms": 26.26,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -3970,7 +3972,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.89,
|
||||
"duration_ms": 25.6,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4022,7 +4024,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 25.67,
|
||||
"duration_ms": 25.76,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -4266,6 +4268,7 @@
|
||||
"artifacts": {
|
||||
"reviewer_kit_json": "reports/output_review_kit.json",
|
||||
"reviewer_kit_markdown": "reports/output_review_kit.md",
|
||||
"reviewer_kit_html": "reports/output_review_kit.html",
|
||||
"blind_pack_json": "reports/output_blind_review_pack.json",
|
||||
"blind_pack_markdown": "reports/output_blind_review_pack.md",
|
||||
"decisions": "reports/output_review_decisions.json",
|
||||
@@ -4740,7 +4743,7 @@
|
||||
"skill_os2_coverage": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"item_count": 20,
|
||||
@@ -10227,7 +10230,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 177,
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
@@ -12600,7 +12603,7 @@
|
||||
"python_compatibility": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"root": ".",
|
||||
"summary": {
|
||||
"target_python": "3.11",
|
||||
@@ -13555,7 +13558,7 @@
|
||||
"architecture_maintainability": {
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"skill_dir": ".",
|
||||
"summary": {
|
||||
"python_file_count": 151,
|
||||
@@ -13588,7 +13591,7 @@
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao.py",
|
||||
"lines": 864,
|
||||
"lines": 867,
|
||||
"kind": "cli-script",
|
||||
"severity": "pass",
|
||||
"recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator."
|
||||
@@ -13609,7 +13612,7 @@
|
||||
},
|
||||
{
|
||||
"path": "scripts/yao_cli_parser.py",
|
||||
"lines": 727,
|
||||
"lines": 728,
|
||||
"kind": "internal-module",
|
||||
"severity": "pass",
|
||||
"recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates."
|
||||
@@ -13672,9 +13675,9 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 751,
|
||||
"other_text_tokens": 1151522,
|
||||
"other_text_tokens": 1154215,
|
||||
"estimated_initial_load_tokens": 944,
|
||||
"estimated_total_text_tokens": 1152273,
|
||||
"estimated_total_text_tokens": 1154966,
|
||||
"relevant_file_count": 490,
|
||||
"unused_resource_dirs": [],
|
||||
"quality_signal_points": 130,
|
||||
@@ -15527,7 +15530,7 @@
|
||||
"adoption_drift": {
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-13T23:14:39Z",
|
||||
"generated_at": "2026-06-13T23:27:35Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
@@ -15551,14 +15554,14 @@
|
||||
},
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -15567,7 +15570,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -15579,31 +15582,31 @@
|
||||
{
|
||||
"skill": "yao-meta-skill",
|
||||
"events": 1,
|
||||
"adoption_events": 0,
|
||||
"accepted": 0,
|
||||
"adoption_events": 1,
|
||||
"accepted": 1,
|
||||
"edited": 0,
|
||||
"rejected": 0,
|
||||
"missed": 0,
|
||||
"adoption_rate": 0
|
||||
"adoption_rate": 100.0
|
||||
}
|
||||
],
|
||||
"next_iteration_candidates": [],
|
||||
"recent_events": [
|
||||
{
|
||||
"command": "unknown",
|
||||
"event": "review_event",
|
||||
"event": "skill_activation",
|
||||
"skill": "yao-meta-skill",
|
||||
"source": "manual",
|
||||
"version": "1.1.0",
|
||||
"activation_type": "manual",
|
||||
"outcome": "reviewed",
|
||||
"activation_type": "explicit",
|
||||
"outcome": "accepted",
|
||||
"failure_type": "none",
|
||||
"timestamp": "2026-06-13T12:00:00Z"
|
||||
"timestamp": "2026-06-13T10:00:00Z"
|
||||
}
|
||||
],
|
||||
"failures": [],
|
||||
"artifacts": {
|
||||
"events_jsonl": "reports/telemetry_events.jsonl",
|
||||
"events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl",
|
||||
"json": "reports/adoption_drift_report.json",
|
||||
"markdown": "reports/adoption_drift_report.md"
|
||||
}
|
||||
@@ -15612,7 +15615,7 @@
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"generated_at": "2026-06-14",
|
||||
"generated_at": "2026-06-13",
|
||||
"summary": {
|
||||
"waiver_count": 0,
|
||||
"active_count": 0,
|
||||
@@ -15662,7 +15665,7 @@
|
||||
"Reviewer links output_review_adjudication or output_execution evidence."
|
||||
],
|
||||
"suggested_evidence": "reports/output_review_adjudication.md",
|
||||
"suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"<reviewer>\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-14 --evidence reports/output_review_adjudication.md",
|
||||
"suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"<reviewer>\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md",
|
||||
"world_class_boundary": "Does not count as provider, human, or public world-class completion evidence."
|
||||
},
|
||||
{
|
||||
@@ -15692,7 +15695,7 @@
|
||||
"schema_version": "1.0",
|
||||
"ok": true,
|
||||
"skill_dir": ".",
|
||||
"source": "reports/review_annotations_input.json",
|
||||
"source": "tests/tmp_review_studio/empty_review_annotations_input.json",
|
||||
"summary": {
|
||||
"annotation_count": 0,
|
||||
"open_count": 0,
|
||||
@@ -15915,7 +15918,7 @@
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"provenance_requirements": [
|
||||
"real external client source",
|
||||
@@ -15943,7 +15946,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -16403,7 +16406,7 @@
|
||||
"intake_errors": [],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
@@ -16438,7 +16441,7 @@
|
||||
"summary": {
|
||||
"ledger_ready_to_claim_world_class": false,
|
||||
"ledger_pending_count": 4,
|
||||
"claim_surface_count": 70,
|
||||
"claim_surface_count": 71,
|
||||
"violation_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
"decision": "claim-guard-pass-evidence-pending"
|
||||
@@ -16627,6 +16630,10 @@
|
||||
"path": "reports/output_review_adjudication.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_review_kit.html",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_review_kit.md",
|
||||
"violation_count": 0
|
||||
@@ -16785,8 +16792,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492",
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -16817,7 +16824,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -16842,7 +16849,7 @@
|
||||
"vscode"
|
||||
],
|
||||
"package_metadata": "registry/packages/yao-meta-skill.json",
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -16865,8 +16872,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_entry_count": 567,
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"archive_entry_count": 568,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -17531,7 +17538,7 @@
|
||||
"installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill",
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 567,
|
||||
"archive_entry_count": 568,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -17873,12 +17880,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -23,7 +23,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 177,
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
},
|
||||
"failures": [],
|
||||
"warnings": [],
|
||||
|
||||
@@ -16,7 +16,7 @@
|
||||
- Interactive scripts: `0`
|
||||
- Package hash scope: `source-contract-without-generated-reports`
|
||||
- Package hash files: `177`
|
||||
- Package SHA256: `09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492`
|
||||
- Package SHA256: `5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045`
|
||||
|
||||
## Failures
|
||||
|
||||
|
||||
@@ -771,7 +771,7 @@
|
||||
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
|
||||
</div>
|
||||
<div class="two-col">
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-0.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-1.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-2.3" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="24.8 75.2" stroke-dashoffset="-3.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="72.1 27.9" stroke-dashoffset="-27.9" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">356项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-0.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-1.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.8 99.2" stroke-dashoffset="-2.3" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="24.8 75.2" stroke-dashoffset="-3.1" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="72.1 27.9" stroke-dashoffset="-27.9" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">357项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
|
||||
<table>
|
||||
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
|
||||
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
|
||||
|
||||
+24
-23
@@ -449,10 +449,10 @@
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 199
|
||||
"file_count": 200
|
||||
}
|
||||
],
|
||||
"file_count": 356,
|
||||
"file_count": 357,
|
||||
"folder_count": 4,
|
||||
"distribution": [
|
||||
{
|
||||
@@ -485,7 +485,7 @@
|
||||
},
|
||||
{
|
||||
"label": "reports",
|
||||
"value": 199
|
||||
"value": 200
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -618,7 +618,7 @@
|
||||
"path": "reports",
|
||||
"label": "Generated evidence and overview artifacts",
|
||||
"kind": "folder",
|
||||
"file_count": 199
|
||||
"file_count": 200
|
||||
}
|
||||
],
|
||||
"strengths": [
|
||||
@@ -882,6 +882,7 @@
|
||||
"artifacts": {
|
||||
"reviewer_kit_json": "reports/output_review_kit.json",
|
||||
"reviewer_kit_markdown": "reports/output_review_kit.md",
|
||||
"reviewer_kit_html": "reports/output_review_kit.html",
|
||||
"blind_pack_json": "reports/output_blind_review_pack.json",
|
||||
"blind_pack_markdown": "reports/output_blind_review_pack.md",
|
||||
"decisions": "reports/output_review_decisions.json",
|
||||
@@ -934,9 +935,9 @@
|
||||
"world_class_task_count": 4,
|
||||
"world_class_ledger_pending_count": 4,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 63
|
||||
"changed_file_count": 33
|
||||
},
|
||||
"commit": "0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b",
|
||||
"commit": "834665b73c1422486be8cb1b1dd826fa58e81be7",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
|
||||
@@ -1040,7 +1041,7 @@
|
||||
"interactive_script_count": 0,
|
||||
"package_hash_scope": "source-contract-without-generated-reports",
|
||||
"package_hash_file_count": 177,
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
},
|
||||
"skill_atlas": {
|
||||
"skill_count": 12,
|
||||
@@ -1078,8 +1079,8 @@
|
||||
"trust_level": "local",
|
||||
"license": "MIT",
|
||||
"checksums": {
|
||||
"package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492",
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
"compatibility": {
|
||||
"openai": "pass",
|
||||
@@ -1110,7 +1111,7 @@
|
||||
},
|
||||
"distribution": {
|
||||
"archive_verified": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"package_verification": "reports/package_verification.json",
|
||||
"install_simulated": true,
|
||||
"install_simulation": "reports/install_simulation.json"
|
||||
@@ -1126,8 +1127,8 @@
|
||||
"target_count": 4,
|
||||
"adapter_count": 4,
|
||||
"archive_present": true,
|
||||
"archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34",
|
||||
"archive_entry_count": 567,
|
||||
"archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7",
|
||||
"archive_entry_count": 568,
|
||||
"failure_count": 0,
|
||||
"warning_count": 0
|
||||
},
|
||||
@@ -1138,7 +1139,7 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"archive_present": true,
|
||||
"archive_entry_count": 567,
|
||||
"archive_entry_count": 568,
|
||||
"archive_extracted": true,
|
||||
"entrypoint_loaded": true,
|
||||
"manifest_loaded": true,
|
||||
@@ -1205,12 +1206,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
}
|
||||
]
|
||||
},
|
||||
@@ -1227,14 +1228,14 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"event_count": 1,
|
||||
"adoption_sample_count": 0,
|
||||
"activation_count": 0,
|
||||
"accepted_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"activation_count": 1,
|
||||
"accepted_count": 1,
|
||||
"edited_count": 0,
|
||||
"rejected_count": 0,
|
||||
"missed_count": 0,
|
||||
"failed_count": 0,
|
||||
"adoption_rate": 0,
|
||||
"adoption_rate": 100.0,
|
||||
"missed_trigger_count": 0,
|
||||
"wrong_trigger_count": 0,
|
||||
"bad_output_count": 0,
|
||||
@@ -1243,7 +1244,7 @@
|
||||
"review_overdue_count": 0,
|
||||
"risk_band": "low",
|
||||
"event_types": {
|
||||
"review_event": 1
|
||||
"skill_activation": 1
|
||||
},
|
||||
"failure_types": {},
|
||||
"source_types": {
|
||||
@@ -1462,7 +1463,7 @@
|
||||
"status": "external_required",
|
||||
"category": "external",
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"runbook": [
|
||||
"python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension://<extension-id>/",
|
||||
@@ -1701,7 +1702,7 @@
|
||||
"owner": "Browser/Chrome/IDE/provider client integrator",
|
||||
"status": "pending",
|
||||
"source_status": "external_required",
|
||||
"current": "external source events 0; adoption samples 0",
|
||||
"current": "external source events 0; adoption samples 1",
|
||||
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
|
||||
"provenance_requirements": [
|
||||
"real external client source",
|
||||
@@ -1729,7 +1730,7 @@
|
||||
],
|
||||
"observed_state": {
|
||||
"external_source_events": 0,
|
||||
"adoption_sample_count": 0,
|
||||
"adoption_sample_count": 1,
|
||||
"raw_content_allowed": false,
|
||||
"risk_band": "low",
|
||||
"accepted": false
|
||||
|
||||
@@ -285,7 +285,7 @@
|
||||
"key": "registry-distribution",
|
||||
"label": "Registry Distribution",
|
||||
"status": "pass",
|
||||
"current": "zip entries 567; install failures 0; permission failures 0",
|
||||
"current": "zip entries 568; install failures 0; permission failures 0",
|
||||
"target": "Package metadata, archive checksum, package verification, and install simulation pass",
|
||||
"evidence": [
|
||||
{
|
||||
|
||||
@@ -27,7 +27,7 @@ Generated at: `2026-06-13`
|
||||
| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. |
|
||||
| Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. |
|
||||
| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. |
|
||||
| Registry Distribution | pass | zip entries 567; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
|
||||
| Registry Distribution | pass | zip entries 568; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
|
||||
| Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. |
|
||||
| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. |
|
||||
| Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
|
||||
|
||||
@@ -196,7 +196,7 @@
|
||||
"label": "Registry Distribution",
|
||||
"status": "pass",
|
||||
"objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.",
|
||||
"current": "archive entries 567; install failures 0",
|
||||
"current": "archive entries 568; install failures 0",
|
||||
"command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .",
|
||||
"test": "python3 tests/verify_registry_audit.py",
|
||||
"evidence": [
|
||||
|
||||
@@ -23,7 +23,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
|
||||
| Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` |
|
||||
| Trust Security | `pass` | 93 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` |
|
||||
| Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` |
|
||||
| Registry Distribution | `pass` | archive entries 567; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
|
||||
| Registry Distribution | `pass` | archive entries 568; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` |
|
||||
| Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` |
|
||||
| Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` |
|
||||
|
||||
|
||||
@@ -70,12 +70,12 @@
|
||||
{
|
||||
"field": "archive_sha256",
|
||||
"from": "",
|
||||
"to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34"
|
||||
"to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7"
|
||||
},
|
||||
{
|
||||
"field": "package_sha256",
|
||||
"from": "0000000000000000000000000000000000000000000000000000000000000000",
|
||||
"to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492"
|
||||
"to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045"
|
||||
}
|
||||
]
|
||||
},
|
||||
|
||||
@@ -6,7 +6,7 @@
|
||||
"summary": {
|
||||
"ledger_ready_to_claim_world_class": false,
|
||||
"ledger_pending_count": 4,
|
||||
"claim_surface_count": 70,
|
||||
"claim_surface_count": 71,
|
||||
"violation_count": 0,
|
||||
"overclaim_guard_active": true,
|
||||
"decision": "claim-guard-pass-evidence-pending"
|
||||
@@ -195,6 +195,10 @@
|
||||
"path": "reports/output_review_adjudication.md",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_review_kit.html",
|
||||
"violation_count": 0
|
||||
},
|
||||
{
|
||||
"path": "reports/output_review_kit.md",
|
||||
"violation_count": 0
|
||||
|
||||
@@ -7,7 +7,7 @@ Generated at: `2026-06-13`
|
||||
- decision: `claim-guard-pass-evidence-pending`
|
||||
- ledger ready to claim world-class: `false`
|
||||
- ledger pending evidence: `4`
|
||||
- claim surfaces scanned: `70`
|
||||
- claim surfaces scanned: `71`
|
||||
- violations: `0`
|
||||
- overclaim guard active: `true`
|
||||
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
"""Prepare a reviewer-facing blind A/B output review kit."""
|
||||
|
||||
import argparse
|
||||
import html
|
||||
import json
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
@@ -22,6 +23,7 @@ DEFAULT_BLIND_PACK_MD = ROOT / "reports" / "output_blind_review_pack.md"
|
||||
DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json"
|
||||
DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_kit.json"
|
||||
DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_kit.md"
|
||||
DEFAULT_OUTPUT_HTML = ROOT / "reports" / "output_review_kit.html"
|
||||
|
||||
|
||||
def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]:
|
||||
@@ -236,12 +238,174 @@ def render_markdown(payload: dict[str, Any]) -> str:
|
||||
return "\n".join(lines).strip() + "\n"
|
||||
|
||||
|
||||
def html_text(value: Any) -> str:
|
||||
return html.escape(str(value or ""), quote=True)
|
||||
|
||||
|
||||
def status_label(status: str) -> str:
|
||||
return {
|
||||
"awaiting-decision": "Awaiting",
|
||||
"needs-fix": "Needs fix",
|
||||
"ready-for-adjudication": "Ready",
|
||||
}.get(status, status)
|
||||
|
||||
|
||||
def render_html_rubric(rubric: list[dict[str, Any]]) -> str:
|
||||
if not rubric:
|
||||
return "<li>No rubric items found.</li>"
|
||||
return "".join(
|
||||
"<li><span>{id}</span><p>{description}</p><small>{weight}</small></li>".format(
|
||||
id=html_text(item.get("id", "")),
|
||||
description=html_text(item.get("description", "")),
|
||||
weight=html_text(item.get("weight", "")),
|
||||
)
|
||||
for item in rubric
|
||||
)
|
||||
|
||||
|
||||
def render_html_cases(cases: list[dict[str, Any]]) -> str:
|
||||
cards = []
|
||||
for index, case in enumerate(cases, start=1):
|
||||
state = case.get("decision_state", {})
|
||||
status = str(state.get("status", "awaiting-decision"))
|
||||
cards.append(
|
||||
f"""
|
||||
<article class="case-card" id="case-{html_text(case.get('case_id', ''))}">
|
||||
<header class="case-head">
|
||||
<div>
|
||||
<span class="case-index">Case {index:02d}</span>
|
||||
<h3>{html_text(case.get('case_id', ''))}</h3>
|
||||
</div>
|
||||
<span class="status-pill {html_text(status)}">{html_text(status_label(status))}</span>
|
||||
</header>
|
||||
<p class="prompt">{html_text(case.get('prompt', ''))}</p>
|
||||
<section class="rubric"><h4>Rubric</h4><ul>{render_html_rubric(case.get('rubric', []))}</ul></section>
|
||||
<section class="variants" aria-label="Blind output variants">
|
||||
<div class="variant"><h4>Variant A</h4><pre>{html_text(case.get('variant_a', {}).get('output', ''))}</pre></div>
|
||||
<div class="variant"><h4>Variant B</h4><pre>{html_text(case.get('variant_b', {}).get('output', ''))}</pre></div>
|
||||
</section>
|
||||
<footer class="case-foot">
|
||||
<span>Winner recorded: {html_text(str(state.get('winner_variant_recorded', False)).lower())}</span>
|
||||
<span>Confidence: {html_text(str(state.get('confidence_recorded', False)).lower())}</span>
|
||||
<span>Reason: {html_text(str(state.get('reason_recorded', False)).lower())}</span>
|
||||
<strong>{html_text(state.get('blocking_reason', ''))}</strong>
|
||||
</footer>
|
||||
</article>"""
|
||||
)
|
||||
return "\n".join(cards)
|
||||
|
||||
|
||||
def decision_template_json(cases: list[dict[str, Any]]) -> str:
|
||||
template = {
|
||||
"schema_version": "1.0",
|
||||
"reviewer": "",
|
||||
"reviewed_at": "",
|
||||
"decisions": [
|
||||
{"case_id": case.get("case_id", ""), "winner_variant": "", "confidence": None, "reason": ""}
|
||||
for case in cases
|
||||
],
|
||||
}
|
||||
return json.dumps(template, ensure_ascii=False, indent=2)
|
||||
|
||||
|
||||
def render_html(payload: dict[str, Any]) -> str:
|
||||
summary = payload["summary"]
|
||||
contract = payload["review_contract"]
|
||||
stats = [
|
||||
("Cases", summary["case_count"]),
|
||||
("Ready", summary["ready_for_adjudication_count"]),
|
||||
("Pending", summary["pending_decision_count"]),
|
||||
("Invalid", summary["invalid_decision_count"]),
|
||||
]
|
||||
stat_html = "".join(f"<div><span>{html_text(label)}</span><strong>{html_text(value)}</strong></div>" for label, value in stats)
|
||||
flow_html = "".join(f"<li>{html_text(step)}</li>" for step in contract["reviewer_steps"])
|
||||
privacy_html = "".join(f"<li>{html_text(item)}</li>" for item in contract["privacy_contract"])
|
||||
decision_json = html_text(decision_template_json(payload["cases"]))
|
||||
return f"""<!doctype html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="utf-8">
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1">
|
||||
<title>Output Review Kit</title>
|
||||
<style>
|
||||
:root {{ --ink:#1B365D; --text:#202124; --muted:#6f6a63; --line:#e8e1d8; --soft:#f8f6f2; --warn:#9b4d0f; --ok:#1f6f43; }}
|
||||
* {{ box-sizing:border-box; }}
|
||||
body {{ margin:0; background:#fff; color:var(--text); font:16px/1.55 Georgia, "Times New Roman", serif; }}
|
||||
.shell {{ max-width:1180px; margin:0 auto; padding:36px 24px 72px; }}
|
||||
.topbar {{ position:sticky; top:0; z-index:10; background:rgba(255,255,255,.96); border-bottom:1px solid var(--line); backdrop-filter:blur(8px); }}
|
||||
.topbar-inner {{ max-width:1180px; margin:0 auto; padding:12px 24px; display:flex; justify-content:space-between; gap:18px; align-items:center; }}
|
||||
.brand {{ color:var(--ink); font-weight:700; letter-spacing:0; }}
|
||||
.links {{ display:flex; gap:12px; flex-wrap:wrap; }}
|
||||
.links a {{ color:var(--ink); text-decoration:none; border-bottom:1px solid transparent; }}
|
||||
.links a:hover {{ border-color:var(--ink); }}
|
||||
.hero {{ border-bottom:1px solid var(--line); padding:34px 0 28px; }}
|
||||
.eyebrow {{ color:var(--ink); text-transform:uppercase; font-size:12px; letter-spacing:0; font-weight:700; }}
|
||||
h1 {{ margin:8px 0 12px; color:var(--ink); font-size:58px; line-height:1.02; letter-spacing:0; }}
|
||||
.lede {{ max-width:760px; color:var(--muted); font-size:20px; }}
|
||||
.stats {{ display:grid; grid-template-columns:repeat(4, minmax(0,1fr)); gap:12px; margin:28px 0; }}
|
||||
.stats div, .panel, .case-card {{ border:1px solid var(--line); border-radius:8px; background:#fff; }}
|
||||
.stats div {{ padding:16px; }}
|
||||
.stats span {{ display:block; color:var(--muted); font-size:13px; }}
|
||||
.stats strong {{ color:var(--ink); font-size:34px; line-height:1; }}
|
||||
.grid {{ display:grid; grid-template-columns:minmax(0,1fr) minmax(280px,.42fr); gap:18px; align-items:start; }}
|
||||
.panel {{ padding:20px; }}
|
||||
h2, h3, h4 {{ color:var(--ink); letter-spacing:0; }}
|
||||
h2 {{ margin:0 0 14px; font-size:28px; }}
|
||||
h3 {{ margin:0; font-size:24px; }}
|
||||
h4 {{ margin:0 0 10px; font-size:16px; }}
|
||||
ol, ul {{ padding-left:22px; }}
|
||||
code, pre {{ font-family:ui-monospace, SFMono-Regular, Menlo, Consolas, monospace; }}
|
||||
pre {{ white-space:pre-wrap; overflow-wrap:anywhere; margin:0; color:#2b2b2b; font-size:13px; line-height:1.5; }}
|
||||
.case-card {{ margin:22px 0; padding:22px; scroll-margin-top:72px; }}
|
||||
.case-head, .case-foot {{ display:flex; gap:14px; justify-content:space-between; align-items:flex-start; flex-wrap:wrap; }}
|
||||
.case-index {{ color:var(--muted); font-size:12px; text-transform:uppercase; letter-spacing:0; }}
|
||||
.status-pill {{ border:1px solid var(--line); border-radius:999px; padding:4px 10px; color:var(--ink); background:var(--soft); font-size:13px; }}
|
||||
.status-pill.ready-for-adjudication {{ color:var(--ok); }}
|
||||
.status-pill.needs-fix {{ color:var(--warn); }}
|
||||
.prompt {{ color:var(--muted); font-size:18px; }}
|
||||
.rubric {{ margin:18px 0; padding:16px; background:var(--soft); border-radius:8px; }}
|
||||
.rubric ul {{ list-style:none; padding:0; margin:0; display:grid; gap:8px; }}
|
||||
.rubric li {{ display:grid; grid-template-columns:160px 1fr 44px; gap:12px; align-items:start; border-top:1px solid var(--line); padding-top:8px; }}
|
||||
.rubric li:first-child {{ border-top:0; padding-top:0; }}
|
||||
.rubric span {{ color:var(--ink); font-weight:700; overflow-wrap:anywhere; }}
|
||||
.rubric p {{ margin:0; }}
|
||||
.rubric small {{ text-align:right; color:var(--muted); }}
|
||||
.variants {{ display:grid; grid-template-columns:1fr 1fr; gap:16px; }}
|
||||
.variant {{ border:1px solid var(--line); border-radius:8px; padding:16px; min-width:0; }}
|
||||
.case-foot {{ margin-top:16px; padding-top:14px; border-top:1px solid var(--line); color:var(--muted); font-size:13px; }}
|
||||
.case-foot strong {{ flex-basis:100%; color:var(--text); font-weight:400; }}
|
||||
.template {{ background:#101820; color:#f7f2e8; border-radius:8px; padding:16px; }}
|
||||
@media (max-width:820px) {{ .stats, .grid, .variants {{ grid-template-columns:1fr; }} .rubric li {{ grid-template-columns:1fr; }} .rubric small {{ text-align:left; }} h1 {{ font-size:38px; }} }}
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
<nav class="topbar"><div class="topbar-inner"><span class="brand">Output Review Kit</span><div class="links"><a href="#flow">Flow</a><a href="#cases">Cases</a><a href="#template">Template</a></div></div></nav>
|
||||
<main class="shell">
|
||||
<section class="hero">
|
||||
<span class="eyebrow">Blind A/B Human Review</span>
|
||||
<h1>Reviewer cockpit for output quality decisions</h1>
|
||||
<p class="lede">Compare visible Variant A and Variant B outputs, fill the decision file, then run adjudication. The answer key is intentionally hidden from this page.</p>
|
||||
<div class="stats">{stat_html}</div>
|
||||
</section>
|
||||
<section class="grid" id="flow">
|
||||
<article class="panel"><h2>Review Flow</h2><ol>{flow_html}</ol></article>
|
||||
<aside class="panel"><h2>Privacy</h2><ul>{privacy_html}</ul></aside>
|
||||
</section>
|
||||
<section id="cases">{render_html_cases(payload["cases"])}</section>
|
||||
<section class="panel" id="template"><h2>Decision Template</h2><p>Use this shape in {html_text(payload['artifacts']['decisions'])}; leave a case blank when the reviewer is not ready.</p><pre class="template">{decision_json}</pre></section>
|
||||
</main>
|
||||
</body>
|
||||
</html>
|
||||
"""
|
||||
|
||||
|
||||
def prepare_output_review_kit(
|
||||
blind_pack_json: Path,
|
||||
blind_pack_md: Path,
|
||||
decisions_path: Path,
|
||||
output_json: Path,
|
||||
output_md: Path,
|
||||
output_html: Path | None = None,
|
||||
write_template: bool = False,
|
||||
) -> dict[str, Any]:
|
||||
blind_pack, failures = load_json(blind_pack_json)
|
||||
@@ -272,6 +436,7 @@ def prepare_output_review_kit(
|
||||
"artifacts": {
|
||||
"reviewer_kit_json": display_path(output_json),
|
||||
"reviewer_kit_markdown": display_path(output_md),
|
||||
"reviewer_kit_html": display_path(output_html) if output_html else "",
|
||||
"blind_pack_json": display_path(blind_pack_json),
|
||||
"blind_pack_markdown": display_path(blind_pack_md),
|
||||
"decisions": display_path(decisions_path),
|
||||
@@ -287,6 +452,9 @@ def prepare_output_review_kit(
|
||||
output_md.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
|
||||
output_md.write_text(render_markdown(payload), encoding="utf-8")
|
||||
if output_html:
|
||||
output_html.parent.mkdir(parents=True, exist_ok=True)
|
||||
output_html.write_text(render_html(payload), encoding="utf-8")
|
||||
return payload
|
||||
|
||||
|
||||
@@ -297,6 +465,7 @@ def main() -> None:
|
||||
parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS))
|
||||
parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON))
|
||||
parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD))
|
||||
parser.add_argument("--output-html", default=str(DEFAULT_OUTPUT_HTML))
|
||||
parser.add_argument("--write-template", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
@@ -306,6 +475,7 @@ def main() -> None:
|
||||
Path(args.decisions).resolve(),
|
||||
Path(args.output_json).resolve(),
|
||||
Path(args.output_md).resolve(),
|
||||
Path(args.output_html).resolve() if args.output_html else None,
|
||||
write_template=args.write_template,
|
||||
)
|
||||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||||
|
||||
@@ -142,13 +142,13 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = {
|
||||
"output-lab": {
|
||||
"summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。",
|
||||
"why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。",
|
||||
"source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md",
|
||||
"source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md",
|
||||
"source_paths": [
|
||||
{"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ["case_id"]},
|
||||
{"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]},
|
||||
{"path": "reports/output_execution_runs.md", "label": "output execution runs", "kind": "report", "patterns": ["# Output Execution"]},
|
||||
{"path": "reports/output_blind_review_pack.md", "label": "blind A/B review pack", "kind": "report", "patterns": ["# Output Blind"]},
|
||||
{"path": "reports/output_review_decisions.json", "label": "review decisions template", "kind": "report", "patterns": ["decision_contract", "winner_variant"]},
|
||||
{"path": "reports/output_review_kit.html", "label": "reviewer cockpit", "kind": "report", "patterns": ["Output Review Kit", "Variant A"]},
|
||||
{"path": "reports/output_review_adjudication.md", "label": "review adjudication", "kind": "report", "patterns": ["# Output Review"]},
|
||||
],
|
||||
"verification": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review",
|
||||
@@ -736,7 +736,7 @@ def render_html(report: dict[str, Any]) -> str:
|
||||
</section>
|
||||
|
||||
<section class="twocol">
|
||||
<div class="panel"><h2>评审方式</h2><p>先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。</p></div>
|
||||
<div class="panel"><h2>评审方式</h2><p>先打开 reports/output_review_kit.html 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。</p></div>
|
||||
<div class="panel"><h2>运行方式</h2><p>reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。</p></div>
|
||||
</section>
|
||||
|
||||
|
||||
@@ -64,6 +64,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]:
|
||||
"output_execution": "reports/output_execution_runs.md",
|
||||
"output_blind_review": "reports/output_blind_review_pack.md",
|
||||
"output_review_kit": "reports/output_review_kit.md",
|
||||
"output_review_kit_html": "reports/output_review_kit.html",
|
||||
"output_review_decisions": "reports/output_review_decisions.json",
|
||||
"output_review_adjudication": "reports/output_review_adjudication.md",
|
||||
"benchmark_reproducibility": "reports/benchmark_reproducibility.md",
|
||||
|
||||
@@ -203,6 +203,7 @@ def command_report(args: argparse.Namespace) -> int:
|
||||
"compiled_targets": "reports/compiled_targets.json",
|
||||
"output_execution": "reports/output_execution_runs.json",
|
||||
"output_review_kit": "reports/output_review_kit.json",
|
||||
"output_review_kit_html": "reports/output_review_kit.html",
|
||||
"output_review_adjudication": "reports/output_review_adjudication.json",
|
||||
"adoption_drift": "reports/adoption_drift_report.json",
|
||||
"telemetry_hooks": "reports/telemetry_hook_recipes.json",
|
||||
@@ -515,6 +516,8 @@ def command_output_review_kit(args: argparse.Namespace) -> int:
|
||||
cmd.extend(["--output-json", args.output_json])
|
||||
if args.output_md:
|
||||
cmd.extend(["--output-md", args.output_md])
|
||||
if args.output_html:
|
||||
cmd.extend(["--output-html", args.output_html])
|
||||
if args.write_template:
|
||||
cmd.append("--write-template")
|
||||
result = run_script("prepare_output_review_kit.py", cmd)
|
||||
|
||||
@@ -596,6 +596,7 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int]
|
||||
output_review_kit_cmd.add_argument("--decisions")
|
||||
output_review_kit_cmd.add_argument("--output-json")
|
||||
output_review_kit_cmd.add_argument("--output-md")
|
||||
output_review_kit_cmd.add_argument("--output-html")
|
||||
output_review_kit_cmd.add_argument("--write-template", action="store_true")
|
||||
output_review_kit_cmd.set_defaults(func=_handler(command_handlers, "command_output_review_kit"))
|
||||
|
||||
|
||||
@@ -67,6 +67,7 @@ def main() -> None:
|
||||
decisions_path = tmp_root / "output_review_decisions.json"
|
||||
kit_json = tmp_root / "output_review_kit.json"
|
||||
kit_md = tmp_root / "output_review_kit.md"
|
||||
kit_html = tmp_root / "output_review_kit.html"
|
||||
pending_proc = run(
|
||||
[
|
||||
str(REVIEW_KIT),
|
||||
@@ -80,6 +81,8 @@ def main() -> None:
|
||||
str(kit_json),
|
||||
"--output-md",
|
||||
str(kit_md),
|
||||
"--output-html",
|
||||
str(kit_html),
|
||||
"--write-template",
|
||||
]
|
||||
)
|
||||
@@ -93,6 +96,7 @@ def main() -> None:
|
||||
assert pending_payload["summary"]["answer_key_path_exposed"] is False, pending_payload
|
||||
assert pending_payload["summary"]["ready_to_run_adjudication"] is False, pending_payload
|
||||
assert "answer_key" not in pending_payload["artifacts"], pending_payload["artifacts"]
|
||||
assert pending_payload["artifacts"]["reviewer_kit_html"].endswith("tests/tmp_output_review_kit/output_review_kit.html"), pending_payload["artifacts"]
|
||||
assert len(pending_payload["cases"]) == 5, pending_payload["cases"]
|
||||
first_case = pending_payload["cases"][0]
|
||||
assert first_case["variant_a"]["output"], first_case
|
||||
@@ -102,12 +106,18 @@ def main() -> None:
|
||||
|
||||
kit_json_text = kit_json.read_text(encoding="utf-8")
|
||||
kit_md_text = kit_md.read_text(encoding="utf-8")
|
||||
kit_html_text = kit_html.read_text(encoding="utf-8")
|
||||
assert "# Output Review Kit" in kit_md_text, kit_md_text
|
||||
assert "answer key path exposed: `false`" in kit_md_text, kit_md_text
|
||||
assert "Variant A" in kit_md_text and "Variant B" in kit_md_text, kit_md_text
|
||||
assert "python3 scripts/yao.py output-review" in kit_md_text, kit_md_text
|
||||
assert "<title>Output Review Kit</title>" in kit_html_text, kit_html_text
|
||||
assert "Reviewer cockpit for output quality decisions" in kit_html_text, kit_html_text
|
||||
assert "Variant A" in kit_html_text and "Variant B" in kit_html_text, kit_html_text
|
||||
assert "Decision Template" in kit_html_text, kit_html_text
|
||||
assert_no_answer_key_leak(kit_json_text)
|
||||
assert_no_answer_key_leak(kit_md_text)
|
||||
assert_no_answer_key_leak(kit_html_text)
|
||||
|
||||
cli_proc = run(
|
||||
[
|
||||
@@ -123,6 +133,8 @@ def main() -> None:
|
||||
str(tmp_root / "cli_output_review_kit.json"),
|
||||
"--output-md",
|
||||
str(tmp_root / "cli_output_review_kit.md"),
|
||||
"--output-html",
|
||||
str(tmp_root / "cli_output_review_kit.html"),
|
||||
]
|
||||
)
|
||||
cli_payload = json.loads(cli_proc.stdout)
|
||||
@@ -130,6 +142,7 @@ def main() -> None:
|
||||
assert cli_payload["summary"]["case_count"] == 5, cli_payload
|
||||
assert cli_payload["summary"]["answer_key_hidden"] is True, cli_payload
|
||||
assert "answer_key" not in cli_payload["artifacts"], cli_payload["artifacts"]
|
||||
assert (tmp_root / "cli_output_review_kit.html").exists(), cli_payload
|
||||
|
||||
template = json.loads(decisions_path.read_text(encoding="utf-8"))
|
||||
filled = {
|
||||
@@ -160,6 +173,8 @@ def main() -> None:
|
||||
str(tmp_root / "ready_output_review_kit.json"),
|
||||
"--output-md",
|
||||
str(tmp_root / "ready_output_review_kit.md"),
|
||||
"--output-html",
|
||||
str(tmp_root / "ready_output_review_kit.html"),
|
||||
]
|
||||
)
|
||||
ready_payload = json.loads(ready_proc.stdout)
|
||||
@@ -188,6 +203,8 @@ def main() -> None:
|
||||
str(tmp_root / "invalid_output_review_kit.json"),
|
||||
"--output-md",
|
||||
str(tmp_root / "invalid_output_review_kit.md"),
|
||||
"--output-html",
|
||||
str(tmp_root / "invalid_output_review_kit.html"),
|
||||
]
|
||||
)
|
||||
invalid_payload = json.loads(invalid_proc.stdout)
|
||||
|
||||
@@ -21,6 +21,7 @@ def main() -> None:
|
||||
shutil.rmtree(tmp_root)
|
||||
tmp_root.mkdir(parents=True, exist_ok=True)
|
||||
subprocess.run([sys.executable, str(ROOT / "scripts" / "run_output_eval.py")], cwd=ROOT, check=True, capture_output=True, text=True)
|
||||
subprocess.run([sys.executable, str(ROOT / "scripts" / "prepare_output_review_kit.py")], cwd=ROOT, check=True, capture_output=True, text=True)
|
||||
subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
@@ -360,6 +361,7 @@ def main() -> None:
|
||||
assert full_payload["evidence_paths"]["compiled_targets"] == "reports/compiled_targets.md", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["output_execution"] == "reports/output_execution_runs.md", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["output_blind_review"] == "reports/output_blind_review_pack.md", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["output_review_kit_html"] == "reports/output_review_kit.html", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["output_review_decisions"] == "reports/output_review_decisions.json", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["output_review_adjudication"] == "reports/output_review_adjudication.md", full_payload["evidence_paths"]
|
||||
assert full_payload["evidence_paths"]["python_compatibility"] == "reports/python_compatibility.md", full_payload["evidence_paths"]
|
||||
@@ -479,7 +481,7 @@ def main() -> None:
|
||||
"reports/output_quality_scorecard.md",
|
||||
"reports/output_execution_runs.md",
|
||||
"reports/output_blind_review_pack.md",
|
||||
"reports/output_review_decisions.json",
|
||||
"reports/output_review_kit.html",
|
||||
"reports/output_review_adjudication.md",
|
||||
}, synthetic_actions
|
||||
assert all(item["exists"] for item in synthetic_actions[0]["source_refs"]), synthetic_actions
|
||||
@@ -494,7 +496,7 @@ def main() -> None:
|
||||
assert "审查闸门" in html, html[:1200]
|
||||
assert "修复动作" in html, html[:3000]
|
||||
assert "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。" in html, html[:9000]
|
||||
assert "reports/output_review_decisions.json" in html, html[:9000]
|
||||
assert "reports/output_review_kit.html" in html, html[:9000]
|
||||
assert "python3 scripts/adjudicate_output_review.py --write-template" in html, html[:9000]
|
||||
assert "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。" in html, html[:9000]
|
||||
assert "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据" in html, html
|
||||
|
||||
Reference in New Issue
Block a user