From 6b71aa2fb1feac8342dc72ceb2a5d9c5e7572130 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Sun, 14 Jun 2026 07:33:39 +0800 Subject: [PATCH] feat: add output review cockpit --- registry/index.json | 2 +- registry/packages/yao-meta-skill.json | 6 +- reports/adoption_drift_report.json | 2 +- reports/architecture_maintainability.json | 4 +- reports/architecture_maintainability.md | 4 +- reports/benchmark_reproducibility.json | 26 +-- reports/benchmark_reproducibility.md | 16 +- reports/context_budget.json | 6 +- reports/install_simulation.json | 2 +- reports/output_execution_runs.json | 20 +- reports/output_execution_runs.md | 20 +- reports/output_review_kit.html | 220 ++++++++++++++++++++++ reports/output_review_kit.json | 1 + reports/package_verification.json | 4 +- reports/package_verification.md | 2 +- reports/registry_audit.json | 8 +- reports/registry_audit.md | 4 +- reports/review-studio.html | 24 +-- reports/review-studio.json | 159 ++++++++-------- reports/security_trust_report.json | 2 +- reports/security_trust_report.md | 2 +- reports/skill-overview.html | 2 +- reports/skill-overview.json | 47 ++--- reports/skill_os2_audit.json | 2 +- reports/skill_os2_audit.md | 2 +- reports/skill_os2_coverage.json | 2 +- reports/skill_os2_coverage.md | 2 +- reports/upgrade_check.json | 4 +- reports/world_class_claim_guard.json | 6 +- reports/world_class_claim_guard.md | 2 +- scripts/prepare_output_review_kit.py | 170 +++++++++++++++++ scripts/render_review_studio.py | 6 +- scripts/review_studio_data.py | 1 + scripts/yao.py | 3 + scripts/yao_cli_parser.py | 1 + tests/verify_output_review_kit.py | 17 ++ tests/verify_review_studio.py | 6 +- 37 files changed, 617 insertions(+), 190 deletions(-) create mode 100644 reports/output_review_kit.html diff --git a/registry/index.json b/registry/index.json index 1ce25585..07f7bb7f 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index b2a50c4f..e74ef2ce 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492", - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index e6077e18..10e99250 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T23:15:43Z", + "generated_at": "2026-06-13T23:31:32Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index f8613e57..136689ba 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -34,7 +34,7 @@ }, { "path": "scripts/yao.py", - "lines": 864, + "lines": 867, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." @@ -55,7 +55,7 @@ }, { "path": "scripts/yao_cli_parser.py", - "lines": 727, + "lines": 728, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 0aeb7b48..2c66594b 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -27,10 +27,10 @@ No file-size hotspots found. | --- | ---: | --- | --- | | `scripts/render_review_studio.py` | `899` | `cli-script` | `pass` | | `tests/verify_yao_cli.py` | `898` | `test` | `pass` | -| `scripts/yao.py` | `864` | `cli-script` | `pass` | +| `scripts/yao.py` | `867` | `cli-script` | `pass` | | `scripts/skill_report_model.py` | `792` | `internal-module` | `pass` | | `scripts/compile_skill.py` | `734` | `cli-script` | `pass` | -| `scripts/yao_cli_parser.py` | `727` | `internal-module` | `pass` | +| `scripts/yao_cli_parser.py` | `728` | `internal-module` | `pass` | | `scripts/optimize_description.py` | `723` | `cli-script` | `pass` | | `scripts/trust_check.py` | `714` | `internal-module` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 42972a54..04402560 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,14 +3,12 @@ "ok": true, "generated_at": "2026-06-14", "skill_dir": ".", - "commit": "0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b", + "commit": "834665b73c1422486be8cb1b1dd826fa58e81be7", "git_status": { "available": true, "dirty": true, - "changed_file_count": 65, + "changed_file_count": 58, "sample": [ - " M evidence/world_class/templates/native-permission-enforcement.intake.json", - " M references/trust-security-method.md", " M registry/index.json", " M registry/packages/yao-meta-skill.json", " M reports/adoption_drift_report.json", @@ -20,7 +18,9 @@ " M reports/benchmark_reproducibility.json", " M reports/benchmark_reproducibility.md", " M reports/compiled_targets.json", - " M reports/context_budget.json" + " M reports/context_budget.json", + " M reports/install_simulation.json", + " M reports/install_simulation.md" ] }, "summary": { @@ -42,7 +42,7 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 65 + "changed_file_count": 58 }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -116,7 +116,7 @@ "path": "reports/output_execution_runs.json", "exists": true, "bytes": 7967, - "sha256": "5c02c5db6c557ba4f595f0616e5792e61e7f54c51b3a32dddfab9f3a771f7795" + "sha256": "94be89b0096ba1890e95f7ecfe2a25f603f507aca699843af5c466a1e6812848" }, { "label": "blind_review", @@ -151,7 +151,7 @@ "path": "reports/security_trust_report.json", "exists": true, "bytes": 95873, - "sha256": "bce52b7b783cf2caa79c470ae6733073a67870703d0b33b47ad3251264196ef7" + "sha256": "e5bbb8250aeeee8b77cac94c7308484d27c90cf7bc241aea4e66791ec565f83c" }, { "label": "python_compatibility", @@ -165,21 +165,21 @@ "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "03edb72bdd407f9ee8ae7c37ce15c03ce115ee7c621d4de446cae6ce56a08705" + "sha256": "3ea8d598452cf8995d403164bf3d204b2cc40b07accb8ea4883486e805c74e31" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19325, - "sha256": "212ddb0a9c5a2becc3bbc64b209d23c018123557cebbfccb78faed08c5572184" + "sha256": "aef6143865fb67bcf2bc9f75f2cba8308deb2498180257c585050b3b46fb1534" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, "bytes": 8604, - "sha256": "69d98943271497a0c497e93a92352c76cda61c77876f9893efd80fdb99383f1e" + "sha256": "eebfba42d71e4082d3c801fd126e59c6401b3f8c00a00625c70eb82f927ebd65" }, { "label": "skill_os2_audit", @@ -213,8 +213,8 @@ "label": "world_class_claim_guard", "path": "reports/world_class_claim_guard.json", "exists": true, - "bytes": 8179, - "sha256": "299a88eb7b47d16761d66980e01ccd8164609e344c3c98bb0b8a67b56cf68edd" + "bytes": 8267, + "sha256": "79c5b29f0ff639189aa65f3487afab13150e95c7fa7a6a5b59309b1e87873c25" } ], "missing_artifacts": [], diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index cc5988ff..0c637964 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,7 +1,7 @@ # Benchmark Reproducibility Generated at: `2026-06-14` -Commit: `0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b` +Commit: `834665b73c1422486be8cb1b1dd826fa58e81be7` Working tree dirty at generation: `true` ## Summary @@ -16,7 +16,7 @@ Working tree dirty at generation: `true` - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- changed files at generation: `65` +- changed files at generation: `58` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. @@ -40,21 +40,21 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `5c02c5db6c55` | +| output_execution | `reports/output_execution_runs.json` | present | `94be89b0096b` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `bce52b7b783c` | +| trust_report | `reports/security_trust_report.json` | present | `e5bbb8250aee` | | python_compatibility | `reports/python_compatibility.json` | present | `e5dd973d0b5c` | -| registry_audit | `reports/registry_audit.json` | present | `03edb72bdd40` | -| package_verification | `reports/package_verification.json` | present | `212ddb0a9c5a` | -| install_simulation | `reports/install_simulation.json` | present | `69d989432714` | +| registry_audit | `reports/registry_audit.json` | present | `3ea8d598452c` | +| package_verification | `reports/package_verification.json` | present | `aef6143865fb` | +| install_simulation | `reports/install_simulation.json` | present | `eebfba42d71e` | | skill_os2_audit | `reports/skill_os2_audit.json` | present | `a1ab5e625fbc` | | world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `02e49b66159d` | | world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `0b52d0d1528b` | | world_class_evidence_intake | `reports/world_class_evidence_intake.json` | present | `5fbfcd35ac6a` | -| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `299a88eb7b47` | +| world_class_claim_guard | `reports/world_class_claim_guard.json` | present | `79c5b29f0ff6` | ## Reproduction Commands diff --git a/reports/context_budget.json b/reports/context_budget.json index 3fa829ad..39d65a35 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1151542, + "other_text_tokens": 1154440, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1152293, - "relevant_file_count": 490, + "estimated_total_text_tokens": 1155191, + "relevant_file_count": 491, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 1a6a3ce4..38051603 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "tests/tmp_review_studio/install-root/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 567, + "archive_entry_count": 574, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 4c4603a9..9c7e4ce3 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.82, + "duration_ms": 28.21, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.2, + "duration_ms": 27.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.79, + "duration_ms": 28.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.16, + "duration_ms": 27.76, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.57, + "duration_ms": 28.5, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.12, + "duration_ms": 28.05, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.61, + "duration_ms": 28.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.9, + "duration_ms": 28.26, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.06, + "duration_ms": 28.66, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.98, + "duration_ms": 28.46, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index a007ed4c..01a20cb2 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 27.82 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 28.2 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 27.79 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.16 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 28.57 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.12 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.61 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 27.9 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.06 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.98 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 28.21 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 27.95 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 28.36 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 27.76 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 28.5 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.05 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.23 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.26 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.66 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.46 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/output_review_kit.html b/reports/output_review_kit.html new file mode 100644 index 00000000..5359c296 --- /dev/null +++ b/reports/output_review_kit.html @@ -0,0 +1,220 @@ + + + + + + Output Review Kit + + + + +
+
+ Blind A/B Human Review +

Reviewer cockpit for output quality decisions

+

Compare visible Variant A and Variant B outputs, fill the decision file, then run adjudication. The answer key is intentionally hidden from this page.

+
Cases5
Ready
Pending5
Invalid
+
+
+

Review Flow

  1. Open reports/output_blind_review_pack.md or this kit and compare Variant A vs Variant B for each case.
  2. Record choices in reports/output_review_decisions.json without opening the answer key.
  3. Use winner_variant A or B, confidence from 0 to 1, and a short reason for every case.
  4. Run python3 scripts/yao.py output-review after choices are recorded.
  5. Refresh python3 scripts/yao.py review-studio . before asking for release approval.
+ +
+
+
+
+
+ Case 01 +

skill-package-contract

+
+ Awaiting +
+

Turn this repeated workflow into a reusable team skill package.

+

Rubric

  • has-entrypoint

    Output names the routeable Skill entrypoint.

    1.0
  • has-interface

    Output includes neutral interface metadata.

    1.0
  • has-report

    Output points reviewer to generated overview report.

    1.0
  • has-resource-boundary

    Output preserves entrypoint/resource separation.

    1.0
+
+

Variant A

Create a routeable SKILL.md, agents/interface.yaml, reports/skill-overview.html, and a focused eval plan. Keep the root entrypoint lean, move durable guidance into references/, scripts into scripts/, and evidence into reports/.
+

Variant B

I can write a prompt for that workflow and include a short checklist.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 02 +

output-eval-expectation

+
+ Awaiting +
+

Upgrade this production skill so we know whether the generated output is better than baseline.

+

Rubric

  • has-baseline-comparison

    Output explicitly compares with-skill and baseline outputs.

    1.0
  • has-assertion-grading

    Output includes objective assertion grading.

    1.0
  • has-scorecard

    Output produces a scorecard report path.

    1.0
  • has-failure-taxonomy

    Output records failure taxonomy.

    1.0
+
+

Variant A

Add Output Eval Lab cases with baseline_output and with_skill_output, run assertion grading, report with-skill vs baseline pass-rate delta, and record failure taxonomy plus next fixes in reports/output_quality_scorecard.md.
+

Variant B

Add more examples and run the trigger eval again.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 03 +

ir-before-packaging

+
+ Awaiting +
+

Package this skill for OpenAI, Claude, Agent Skills, and generic targets.

+

Rubric

  • has-ir-first

    Output requires Skill IR before packaging.

    1.0
  • has-core-ir-fields

    Output lists core IR contract fields.

    1.0
  • has-targets

    Output names requested runtime targets.

    1.0
  • has-semantic-preservation

    Output says targets must preserve the capability contract.

    1.0
+
+

Variant A

Export folders for each platform and adjust files as needed.
+

Variant B

Export Skill IR first with job_to_be_done, trigger_surface, workflow, resources, eval_plan, risk, and governance. Then compile or package targets from the IR so OpenAI, Claude, Agent Skills, and generic packages preserve the same capability contract.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 04 +

near-neighbor-boundary

+
+ Awaiting +
+

I only need a one-off summary of these notes, not a reusable process.

+

Rubric

  • declines-skill

    Output refuses unnecessary skill creation for one-off work.

    1.0
  • names-near-neighbor

    Output labels the request as near-neighbor instead of owned work.

    1.0
  • requires-reuse-signal

    Output asks for repeat-use evidence before packaging.

    1.0
+
+

Variant A

Create a SKILL.md and a reusable workflow anyway so future notes can use it.
+

Variant B

Do not create a skill for this one-off request. Treat it as a near-neighbor: answer the summary directly unless the user confirms repeated use, shared ownership, or a reusable output contract.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+ +
+
+
+ Case 05 +

file-backed-governed-package

+
+ Awaiting +
+

Turn the attached release brief source into a governed skill package.

+

Rubric

  • uses-file-backed-evidence

    Output names file-backed source evidence.

    1.0
  • has-governance

    Output preserves governed ownership metadata.

    1.0
  • has-output-contract

    Output preserves output and rollback boundaries.

    1.0
  • has-trust-and-scorecard

    Output requires trust and output scorecard artifacts.

    1.0
  • does-not-invent-evidence

    Output forbids invented launch evidence.

    1.0
+
+

Variant A

Draft a release announcement with the changelog, support notes, and owner name.
+

Variant B

Use the file-backed fixture as source evidence, then create a governed skill package with SKILL.md, agents/interface.yaml, owner, review cadence, input_files, output contract, rollback boundary, trust report, and reports/output_quality_scorecard.md. Mark missing launch metrics as missing evidence instead of inventing them.
+
+
+ Winner recorded: false + Confidence: false + Reason: false + Decision template exists but this row is still blank. +
+
+

Decision Template

Use this shape in reports/output_review_decisions.json; leave a case blank when the reviewer is not ready.

{
+  "schema_version": "1.0",
+  "reviewer": "",
+  "reviewed_at": "",
+  "decisions": [
+    {
+      "case_id": "skill-package-contract",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "output-eval-expectation",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "ir-before-packaging",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "near-neighbor-boundary",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    },
+    {
+      "case_id": "file-backed-governed-package",
+      "winner_variant": "",
+      "confidence": null,
+      "reason": ""
+    }
+  ]
+}
+
+ + diff --git a/reports/output_review_kit.json b/reports/output_review_kit.json index 196708b4..93d431ab 100644 --- a/reports/output_review_kit.json +++ b/reports/output_review_kit.json @@ -15,6 +15,7 @@ "artifacts": { "reviewer_kit_json": "reports/output_review_kit.json", "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", "blind_pack_json": "reports/output_blind_review_pack.json", "blind_pack_markdown": "reports/output_blind_review_pack.md", "decisions": "reports/output_review_decisions.json", diff --git a/reports/package_verification.json b/reports/package_verification.json index 33a2630a..7b4b5436 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", - "archive_entry_count": 567, + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", + "archive_entry_count": 568, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 9d243514..c97e9fde 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34` +- Archive SHA256: `fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 0fc5bf26..df0a8ed8 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492", - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index b85fb10d..adb4f690 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492` -- Archive SHA256: `2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34` +- Package SHA256: `5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045` +- Archive SHA256: `fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 6131ea79..248e2d6e 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -434,12 +434,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

93 scripts scanned; secrets found

Py Compat0

154 files scanned for Python 3.11

Arch Debt0

899 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

70 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

567 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review Kit0/5

pending 5; answer key hidden

Review A/B0/5

adjudication decisions; pending 5

Blueprint20/20

2.0 coverage; evidence pending 4

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native; 4 installer-enforced

Trust0

93 scripts scanned; secrets found

Py Compat0

154 files scanned for Python 3.11

Arch Debt0

899 largest lines; 34 CLI handlers

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Intake4/4

0 valid submissions; 0 invalid

Claim Guard0

71 public surfaces scanned

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

568 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 93 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 154 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

151 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 93 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

Python 兼容

Python 3.11; 154 files; 0 compatibility issues; 0 syntax; 0 f-string 3.11 hazards

reports/python_compatibility.json 证据
通过

架构维护

151 Python files; 0 hotspots; 0 blockers; largest 899 lines; 34 CLI handlers

reports/architecture_maintainability.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; installer 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
关注

世界证据

4 pending world-class evidence entries; 1 human pending; 3 external pending; overclaim guard true

reports/world_class_evidence_ledger.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
@@ -449,7 +449,7 @@

修复动作

-
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .
+
关注

输出实验

补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。

没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。
修复位置
evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md
验证命令
python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review
关注

人工批准

对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。

warning 可以被接受，但必须可审计、会过期，并且不能掩盖 blocker。
修复位置
reports/review_waivers.md
验证命令
python3 scripts/render_review_waivers.py .
关注

世界证据

补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据，或明确本次发布不声明 world-class 完成。

世界级结论必须来自已接受的外部/人工证据；计划、metadata fallback、待评审和本地命令都不能替代完成证据。
修复位置
reports/world_class_evidence_ledger.md + reports/world_class_evidence_plan.md + reports/world_class_evidence_intake.md + reports/world_class_submission_review.md
验证命令
python3 scripts/yao.py world-class-submission-review . && python3 scripts/yao.py world-class-ledger . && python3 scripts/yao.py review-studio .
@@ -465,7 +465,7 @@

证据路径

- +
@@ -491,7 +491,7 @@
-

评审方式

先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

+

评审方式

先打开 reports/output_review_kit.html 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

运行方式

reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。

@@ -506,7 +506,7 @@
-

信任报告

Secret
0
脚本数
93
网络脚本
3
Help 失败
0
包体哈希
09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492
+

信任报告

Secret
0
脚本数
93
网络脚本
3
Help 失败
0
包体哈希
5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -531,8 +531,8 @@
-

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

-

漂移信号

事件数
1
采用率
0
漏触发
0
Bad Output Count
0
风险带
low
+

运营回路

1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low

+

漂移信号

事件数
1
采用率
100
漏触发
0
Bad Output Count
0
风险带
low
@@ -543,7 +543,7 @@

世界证据

这里列出每个 world-class 证据项的当前状态、完成定义、证据来源、隐私约束和下一步；计划、metadata fallback、待评审和本地命令不会被当成完成证据。

-
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 0
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 0; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
+
待补证 · external

Provider Holdout

Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.

负责人
operator with provider credentials
当前状态
model-executed 0; token-observed 0
下一步
Run provider-backed holdout cases with real credentials and commit only aggregate evidence.
观测值
model_executed_count: 0; timing_observed_count: 10; token_observed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/provider-holdout.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_execution_runs.json summary.model_executed_count > 0
  • reports/output_execution_runs.json summary.timing_observed_count > 0
  • reports/output_execution_runs.json summary.token_observed_count > 0
  • reports/skill_os2_audit.json item provider-holdout status becomes pass

证据来源

  • reports/output_execution_runs.json
  • reports/output_execution_runs.md
  • reports/skill_os2_audit.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/provider-holdout.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not commit provider credentials or environment dumps.
  • The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
待补证 · human

Human Adjudication

Record real blind A/B reviewer decisions before claiming human output review completion.

负责人
human reviewer
当前状态
0/5 decisions; pending 5
下一步
Record real A/B choices in the decision template, then regenerate adjudication.
观测值
pair_count: 5; judgment_count: 0; pending_count: 5; invalid_decision_count: 0; answer_revealed_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/human-adjudication.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/output_review_adjudication.json summary.pending_count == 0
  • reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
  • reports/output_review_adjudication.json summary.invalid_decision_count == 0
  • reports/skill_os2_audit.json item human-adjudication status becomes pass

证据来源

  • reports/output_blind_review_pack.md
  • reports/output_review_kit.md
  • reports/output_review_decisions.json
  • reports/output_review_adjudication.json
  • reports/output_review_adjudication.md
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/human-adjudication.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Reviewer decisions should not include raw user data or private customer detail.
  • Keep the answer key separate until after decisions are recorded.
待补证 · external

Native Permission Enforcement

Prove at least one real target client or external installer runtime guard enforces approved high-permission capabilities.

负责人
target client or installer integrator
当前状态
native-enforced targets 0; installer-enforced targets 4
下一步
Integrate a real target-client or external installer runtime guard before claiming native permission enforcement.
观测值
native_enforcement_count: 0; metadata_fallback_count: 4; installer_enforcement_pass_count: 4; installer_permission_failure_count: 0; installer_enforcement_ready: True; residual_risk_count: 4; failure_count: 0; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-permission-enforcement.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/runtime_permission_probes.json summary.native_enforcement_count > 0
  • reports/runtime_permission_probes.json summary.failure_count == 0
  • reports/runtime_permission_probes.json summary.installer_enforcement_pass_count records local installer enforcement but does not replace native evidence
  • reports/skill_os2_audit.json item native-permission-enforcement status becomes pass

证据来源

  • dist/targets/*/adapter.json
  • reports/runtime_permission_probes.json
  • reports/runtime_permission_probes.md
  • reports/install_simulation.json
  • reports/install_simulation.md
  • security/permission_policy.json
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-permission-enforcement.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Do not mark native_enforcement true for metadata-only fallbacks.
  • Keep residual risks visible for targets that still rely on operator enforcement.
待补证 · external

Native Client Telemetry

Import production metadata-only events from a real external client into the local drift loop.

负责人
Browser/Chrome/IDE/provider client integrator
当前状态
external source events 0; adoption samples 1
下一步
Install a real client against the native host and import production metadata-only events.
观测值
external_source_events: 0; adoption_sample_count: 1; raw_content_allowed: False; risk_band: low; accepted: False
提交态
status: missing; path: evidence/world_class/submissions/native-client-telemetry.json; attested_real_evidence: False; privacy_contract_satisfied: False

完成定义

  • reports/adoption_drift_report.json summary.source_types.external > 0
  • reports/adoption_drift_report.json summary.adoption_sample_count > 0
  • reports/skill_os2_audit.json item native-client-telemetry status becomes pass

证据来源

  • reports/adoption_drift_report.json
  • reports/adoption_drift_report.md
  • reports/telemetry_hook_recipes.json
  • scripts/telemetry_native_host.py
  • evidence/world_class/intake.schema.json
  • evidence/world_class/templates/native-client-telemetry.intake.json
  • reports/world_class_evidence_intake.json
  • reports/world_class_evidence_intake.md

隐私约束

  • Telemetry must remain metadata-only and local-first.
  • Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
@@ -568,18 +568,18 @@
-

声明守卫

台账可声明
否
台账待补
4
声明面
70
违规数
0
Overclaim Guard Active
是
+

声明守卫

台账可声明
否
台账待补
4
声明面
71
违规数
0
Overclaim Guard Active
是

声明边界

claim guard 扫描 README、docs 和 reports 中的完成态表述；ledger 未 ready 时，任何英文完成断言、true 状态声明或中文完成态都会阻断发布审查。

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
567
失败数
0
警告数
0
归档哈希
2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
568
失败数
0
警告数
0
归档哈希
fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7
diff --git a/reports/review-studio.json b/reports/review-studio.json index 9e9d1dee..7b65daa1 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -107,7 +107,7 @@ "key": "operations-loop", "label": "运营回路", "status": "pass", - "detail": "1 metadata events; adoption 0; missed 0; bad-output 0; risk low", + "detail": "1 metadata events; adoption 100.0; missed 0; bad-output 0; risk low", "evidence": "reports/adoption_drift_report.json", "link": "adoption_drift_report.md" }, @@ -179,7 +179,7 @@ "priority": "warning", "summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。", "why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", - "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md", "source_refs": [ { "path": "evals/output/cases.jsonl", @@ -214,12 +214,12 @@ "link": "output_blind_review_pack.md" }, { - "path": "reports/output_review_decisions.json", - "label": "review decisions template", + "path": "reports/output_review_kit.html", + "label": "reviewer cockpit", "kind": "report", - "line": 5, + "line": 6, "exists": true, - "link": "output_review_decisions.json" + "link": "output_review_kit.html" }, { "path": "reports/output_review_adjudication.md", @@ -382,6 +382,7 @@ "output_execution": "reports/output_execution_runs.md", "output_blind_review": "reports/output_blind_review_pack.md", "output_review_kit": "reports/output_review_kit.md", + "output_review_kit_html": "reports/output_review_kit.html", "output_review_decisions": "reports/output_review_decisions.json", "output_review_adjudication": "reports/output_review_adjudication.md", "benchmark_reproducibility": "reports/benchmark_reproducibility.md", @@ -860,10 +861,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 199 + "file_count": 200 } ], - "file_count": 356, + "file_count": 357, "folder_count": 4, "distribution": [ { @@ -896,7 +897,7 @@ }, { "label": "reports", - "value": 199 + "value": 200 } ] }, @@ -1029,7 +1030,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 199 + "file_count": 200 } ], "strengths": [ @@ -1293,6 +1294,7 @@ "artifacts": { "reviewer_kit_json": "reports/output_review_kit.json", "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", "blind_pack_json": "reports/output_blind_review_pack.json", "blind_pack_markdown": "reports/output_blind_review_pack.md", "decisions": "reports/output_review_decisions.json", @@ -1345,9 +1347,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 63 + "changed_file_count": 33 }, - "commit": "0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b", + "commit": "834665b73c1422486be8cb1b1dd826fa58e81be7", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1451,7 +1453,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 177, - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" }, "skill_atlas": { "skill_count": 12, @@ -1489,8 +1491,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492", - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, "compatibility": { "openai": "pass", @@ -1521,7 +1523,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1537,8 +1539,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", - "archive_entry_count": 567, + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", + "archive_entry_count": 568, "failure_count": 0, "warning_count": 0 }, @@ -1549,7 +1551,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 567, + "archive_entry_count": 568, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1616,12 +1618,12 @@ { "field": "archive_sha256", "from": "", - "to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" } ] }, @@ -1638,14 +1640,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1654,7 +1656,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -1873,7 +1875,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -2112,7 +2114,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ "real external client source", @@ -2140,7 +2142,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -3790,7 +3792,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.46, + "duration_ms": 25.88, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3818,7 +3820,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.0, + "duration_ms": 25.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3841,7 +3843,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.44, + "duration_ms": 25.73, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3869,7 +3871,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.47, + "duration_ms": 26.83, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3892,7 +3894,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.76, + "duration_ms": 28.58, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3920,7 +3922,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.7, + "duration_ms": 27.09, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3943,7 +3945,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.55, + "duration_ms": 26.26, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3970,7 +3972,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.89, + "duration_ms": 25.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4022,7 +4024,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.67, + "duration_ms": 25.76, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4266,6 +4268,7 @@ "artifacts": { "reviewer_kit_json": "reports/output_review_kit.json", "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", "blind_pack_json": "reports/output_blind_review_pack.json", "blind_pack_markdown": "reports/output_blind_review_pack.md", "decisions": "reports/output_review_decisions.json", @@ -4740,7 +4743,7 @@ "skill_os2_coverage": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "item_count": 20, @@ -10227,7 +10230,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 177, - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" }, "failures": [], "warnings": [], @@ -12600,7 +12603,7 @@ "python_compatibility": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "root": ".", "summary": { "target_python": "3.11", @@ -13555,7 +13558,7 @@ "architecture_maintainability": { "schema_version": "1.0", "ok": true, - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "skill_dir": ".", "summary": { "python_file_count": 151, @@ -13588,7 +13591,7 @@ }, { "path": "scripts/yao.py", - "lines": 864, + "lines": 867, "kind": "cli-script", "severity": "pass", "recommendation": "Split command handlers by domain while keeping scripts/yao.py as the thin CLI orchestrator." @@ -13609,7 +13612,7 @@ }, { "path": "scripts/yao_cli_parser.py", - "lines": 727, + "lines": 728, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -13672,9 +13675,9 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 1151522, + "other_text_tokens": 1154215, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 1152273, + "estimated_total_text_tokens": 1154966, "relevant_file_count": 490, "unused_resource_dirs": [], "quality_signal_points": 130, @@ -15527,7 +15530,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T23:14:39Z", + "generated_at": "2026-06-13T23:27:35Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -15551,14 +15554,14 @@ }, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -15567,7 +15570,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -15579,31 +15582,31 @@ { "skill": "yao-meta-skill", "events": 1, - "adoption_events": 0, - "accepted": 0, + "adoption_events": 1, + "accepted": 1, "edited": 0, "rejected": 0, "missed": 0, - "adoption_rate": 0 + "adoption_rate": 100.0 } ], "next_iteration_candidates": [], "recent_events": [ { "command": "unknown", - "event": "review_event", + "event": "skill_activation", "skill": "yao-meta-skill", "source": "manual", "version": "1.1.0", - "activation_type": "manual", - "outcome": "reviewed", + "activation_type": "explicit", + "outcome": "accepted", "failure_type": "none", - "timestamp": "2026-06-13T12:00:00Z" + "timestamp": "2026-06-13T10:00:00Z" } ], "failures": [], "artifacts": { - "events_jsonl": "reports/telemetry_events.jsonl", + "events_jsonl": "tests/tmp_review_studio/telemetry_events.jsonl", "json": "reports/adoption_drift_report.json", "markdown": "reports/adoption_drift_report.md" } @@ -15612,7 +15615,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "generated_at": "2026-06-14", + "generated_at": "2026-06-13", "summary": { "waiver_count": 0, "active_count": 0, @@ -15662,7 +15665,7 @@ "Reviewer links output_review_adjudication or output_execution evidence." ], "suggested_evidence": "reports/output_review_adjudication.md", - "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-14 --evidence reports/output_review_adjudication.md", + "suggested_command": "python3 scripts/yao.py review-waivers . --add-waiver --gate-key output-lab --reviewer \"\" --reason \"Output Lab has pending human/provider evidence; accepted only for this bounded review scope.\" --expires-at 2027-06-13 --evidence reports/output_review_adjudication.md", "world_class_boundary": "Does not count as provider, human, or public world-class completion evidence." }, { @@ -15692,7 +15695,7 @@ "schema_version": "1.0", "ok": true, "skill_dir": ".", - "source": "reports/review_annotations_input.json", + "source": "tests/tmp_review_studio/empty_review_annotations_input.json", "summary": { "annotation_count": 0, "open_count": 0, @@ -15915,7 +15918,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ "real external client source", @@ -15943,7 +15946,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -16403,7 +16406,7 @@ "intake_errors": [], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false @@ -16438,7 +16441,7 @@ "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 70, + "claim_surface_count": 71, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -16627,6 +16630,10 @@ "path": "reports/output_review_adjudication.md", "violation_count": 0 }, + { + "path": "reports/output_review_kit.html", + "violation_count": 0 + }, { "path": "reports/output_review_kit.md", "violation_count": 0 @@ -16785,8 +16792,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492", - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, "compatibility": { "openai": "pass", @@ -16817,7 +16824,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -16842,7 +16849,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" } ] }, @@ -16865,8 +16872,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", - "archive_entry_count": 567, + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", + "archive_entry_count": 568, "failure_count": 0, "warning_count": 0 }, @@ -17531,7 +17538,7 @@ "installed_skill_dir": "dist/install-simulation/simulate-yao-meta-skill/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 567, + "archive_entry_count": 568, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -17873,12 +17880,12 @@ { "field": "archive_sha256", "from": "", - "to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 0a5b0c72..80033bd9 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 177, - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 1ad8e480..a8830fed 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `177` -- Package SHA256: `09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492` +- Package SHA256: `5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045` ## Failures diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 35656b06..6e6c072b 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布356项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布357项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 9c8acff2..24adf3af 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -449,10 +449,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 199 + "file_count": 200 } ], - "file_count": 356, + "file_count": 357, "folder_count": 4, "distribution": [ { @@ -485,7 +485,7 @@ }, { "label": "reports", - "value": 199 + "value": 200 } ] }, @@ -618,7 +618,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 199 + "file_count": 200 } ], "strengths": [ @@ -882,6 +882,7 @@ "artifacts": { "reviewer_kit_json": "reports/output_review_kit.json", "reviewer_kit_markdown": "reports/output_review_kit.md", + "reviewer_kit_html": "reports/output_review_kit.html", "blind_pack_json": "reports/output_blind_review_pack.json", "blind_pack_markdown": "reports/output_blind_review_pack.md", "decisions": "reports/output_review_decisions.json", @@ -934,9 +935,9 @@ "world_class_task_count": 4, "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 63 + "changed_file_count": 33 }, - "commit": "0cf7f32e32e2b5654a4d03cf91f025446e6d5b7b", + "commit": "834665b73c1422486be8cb1b1dd826fa58e81be7", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1040,7 +1041,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 177, - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" }, "skill_atlas": { "skill_count": 12, @@ -1078,8 +1079,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492", - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "package_sha256": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, "compatibility": { "openai": "pass", @@ -1110,7 +1111,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1126,8 +1127,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34", - "archive_entry_count": 567, + "archive_sha256": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7", + "archive_entry_count": 568, "failure_count": 0, "warning_count": 0 }, @@ -1138,7 +1139,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 567, + "archive_entry_count": 568, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1205,12 +1206,12 @@ { "field": "archive_sha256", "from": "", - "to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" } ] }, @@ -1227,14 +1228,14 @@ "ok": true, "summary": { "event_count": 1, - "adoption_sample_count": 0, - "activation_count": 0, - "accepted_count": 0, + "adoption_sample_count": 1, + "activation_count": 1, + "accepted_count": 1, "edited_count": 0, "rejected_count": 0, "missed_count": 0, "failed_count": 0, - "adoption_rate": 0, + "adoption_rate": 100.0, "missed_trigger_count": 0, "wrong_trigger_count": 0, "bad_output_count": 0, @@ -1243,7 +1244,7 @@ "review_overdue_count": 0, "risk_band": "low", "event_types": { - "review_event": 1 + "skill_activation": 1 }, "failure_types": {}, "source_types": { @@ -1462,7 +1463,7 @@ "status": "external_required", "category": "external", "owner": "Browser/Chrome/IDE/provider client integrator", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "runbook": [ "python3 scripts/telemetry_native_host.py . --write-launcher /tmp/yao-telemetry-host.sh --write-manifest /tmp/yao-telemetry-host.json --allowed-origin chrome-extension:///", @@ -1701,7 +1702,7 @@ "owner": "Browser/Chrome/IDE/provider client integrator", "status": "pending", "source_status": "external_required", - "current": "external source events 0; adoption samples 0", + "current": "external source events 0; adoption samples 1", "objective": "Import production metadata-only events from a real external client into the local drift loop.", "provenance_requirements": [ "real external client source", @@ -1729,7 +1730,7 @@ ], "observed_state": { "external_source_events": 0, - "adoption_sample_count": 0, + "adoption_sample_count": 1, "raw_content_allowed": false, "risk_band": "low", "accepted": false diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 4721235d..7b0a8f44 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -285,7 +285,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 567; install failures 0; permission failures 0", + "current": "zip entries 568; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 4320445c..414db533 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -27,7 +27,7 @@ Generated at: `2026-06-13` | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 567; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 568; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 3; score 91 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 1 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 32e87dcf..e4766103 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -196,7 +196,7 @@ "label": "Registry Distribution", "status": "pass", "objective": "Skill packages are installable, versioned, checksumed, and upgrade-reviewable.", - "current": "archive entries 567; install failures 0", + "current": "archive entries 568; install failures 0", "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .", "test": "python3 tests/verify_registry_audit.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index 868cbbef..d2a8c5dd 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -23,7 +23,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | | Trust Security | `pass` | 93 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | -| Registry Distribution | `pass` | archive entries 567; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | +| Registry Distribution | `pass` | archive entries 568; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | | Telemetry Drift | `pass` | events 1; recipes 5; risk low | `python3 scripts/yao.py telemetry-hooks . && python3 scripts/yao.py adoption-drift .` | `python3 tests/verify_telemetry_hooks.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 092dbf3d..ba5be63e 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "2fbe2e4a06aa71dacd59865cf34adb163940972d04316ebbeb479a83cb46bd34" + "to": "fc334a4b4da4af6c463510a76551e018633deb5626dca38d9aaaa2c636591eb7" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "09013a0df3555b5f1ef58254c1bc78ebd2383f8efbfe67a7a9c1010a172fb492" + "to": "5b6eb1631e77feb6fe3f4199e128607cdc719f8ec0675c92ae986aad2553a045" } ] }, diff --git a/reports/world_class_claim_guard.json b/reports/world_class_claim_guard.json index 114c8f78..f02c3351 100644 --- a/reports/world_class_claim_guard.json +++ b/reports/world_class_claim_guard.json @@ -6,7 +6,7 @@ "summary": { "ledger_ready_to_claim_world_class": false, "ledger_pending_count": 4, - "claim_surface_count": 70, + "claim_surface_count": 71, "violation_count": 0, "overclaim_guard_active": true, "decision": "claim-guard-pass-evidence-pending" @@ -195,6 +195,10 @@ "path": "reports/output_review_adjudication.md", "violation_count": 0 }, + { + "path": "reports/output_review_kit.html", + "violation_count": 0 + }, { "path": "reports/output_review_kit.md", "violation_count": 0 diff --git a/reports/world_class_claim_guard.md b/reports/world_class_claim_guard.md index 61e05981..cc998c5a 100644 --- a/reports/world_class_claim_guard.md +++ b/reports/world_class_claim_guard.md @@ -7,7 +7,7 @@ Generated at: `2026-06-13` - decision: `claim-guard-pass-evidence-pending` - ledger ready to claim world-class: `false` - ledger pending evidence: `4` -- claim surfaces scanned: `70` +- claim surfaces scanned: `71` - violations: `0` - overclaim guard active: `true` diff --git a/scripts/prepare_output_review_kit.py b/scripts/prepare_output_review_kit.py index 283568c3..4c5ff5ab 100644 --- a/scripts/prepare_output_review_kit.py +++ b/scripts/prepare_output_review_kit.py @@ -2,6 +2,7 @@ """Prepare a reviewer-facing blind A/B output review kit.""" import argparse +import html import json from pathlib import Path from typing import Any @@ -22,6 +23,7 @@ DEFAULT_BLIND_PACK_MD = ROOT / "reports" / "output_blind_review_pack.md" DEFAULT_DECISIONS = ROOT / "reports" / "output_review_decisions.json" DEFAULT_OUTPUT_JSON = ROOT / "reports" / "output_review_kit.json" DEFAULT_OUTPUT_MD = ROOT / "reports" / "output_review_kit.md" +DEFAULT_OUTPUT_HTML = ROOT / "reports" / "output_review_kit.html" def load_optional_decisions(path: Path) -> tuple[dict[str, Any], list[str]]: @@ -236,12 +238,174 @@ def render_markdown(payload: dict[str, Any]) -> str: return "\n".join(lines).strip() + "\n" +def html_text(value: Any) -> str: + return html.escape(str(value or ""), quote=True) + + +def status_label(status: str) -> str: + return { + "awaiting-decision": "Awaiting", + "needs-fix": "Needs fix", + "ready-for-adjudication": "Ready", + }.get(status, status) + + +def render_html_rubric(rubric: list[dict[str, Any]]) -> str: + if not rubric: + return "
  • No rubric items found.
    • " + return "".join( + "
  • {id}

    {description}

    {weight}
    • ".format( + id=html_text(item.get("id", "")), + description=html_text(item.get("description", "")), + weight=html_text(item.get("weight", "")), + ) + for item in rubric + ) + + +def render_html_cases(cases: list[dict[str, Any]]) -> str: + cards = [] + for index, case in enumerate(cases, start=1): + state = case.get("decision_state", {}) + status = str(state.get("status", "awaiting-decision")) + cards.append( + f""" +
    +
    +
    + Case {index:02d} +

    {html_text(case.get('case_id', ''))}

    +
    + {html_text(status_label(status))} +
    +

    {html_text(case.get('prompt', ''))}

    +

    Rubric

      {render_html_rubric(case.get('rubric', []))}
    +
    +

    Variant A

    {html_text(case.get('variant_a', {}).get('output', ''))}
    +

    Variant B

    {html_text(case.get('variant_b', {}).get('output', ''))}
    +
    +
    + Winner recorded: {html_text(str(state.get('winner_variant_recorded', False)).lower())} + Confidence: {html_text(str(state.get('confidence_recorded', False)).lower())} + Reason: {html_text(str(state.get('reason_recorded', False)).lower())} + {html_text(state.get('blocking_reason', ''))} +
    +
    """ + ) + return "\n".join(cards) + + +def decision_template_json(cases: list[dict[str, Any]]) -> str: + template = { + "schema_version": "1.0", + "reviewer": "", + "reviewed_at": "", + "decisions": [ + {"case_id": case.get("case_id", ""), "winner_variant": "", "confidence": None, "reason": ""} + for case in cases + ], + } + return json.dumps(template, ensure_ascii=False, indent=2) + + +def render_html(payload: dict[str, Any]) -> str: + summary = payload["summary"] + contract = payload["review_contract"] + stats = [ + ("Cases", summary["case_count"]), + ("Ready", summary["ready_for_adjudication_count"]), + ("Pending", summary["pending_decision_count"]), + ("Invalid", summary["invalid_decision_count"]), + ] + stat_html = "".join(f"
    {html_text(label)}{html_text(value)}
    " for label, value in stats) + flow_html = "".join(f"
  • {html_text(step)}
    • " for step in contract["reviewer_steps"]) + privacy_html = "".join(f"
  • {html_text(item)}
    • " for item in contract["privacy_contract"]) + decision_json = html_text(decision_template_json(payload["cases"])) + return f""" + + + + + Output Review Kit + + + + +
    +
    + Blind A/B Human Review +

    Reviewer cockpit for output quality decisions

    +

    Compare visible Variant A and Variant B outputs, fill the decision file, then run adjudication. The answer key is intentionally hidden from this page.

    +
    {stat_html}
    +
    +
    +

    Review Flow

      {flow_html}
    + +
    +
    {render_html_cases(payload["cases"])}
    +

    Decision Template

    Use this shape in {html_text(payload['artifacts']['decisions'])}; leave a case blank when the reviewer is not ready.

    {decision_json}
    +
    + + +""" + + def prepare_output_review_kit( blind_pack_json: Path, blind_pack_md: Path, decisions_path: Path, output_json: Path, output_md: Path, + output_html: Path | None = None, write_template: bool = False, ) -> dict[str, Any]: blind_pack, failures = load_json(blind_pack_json) @@ -272,6 +436,7 @@ def prepare_output_review_kit( "artifacts": { "reviewer_kit_json": display_path(output_json), "reviewer_kit_markdown": display_path(output_md), + "reviewer_kit_html": display_path(output_html) if output_html else "", "blind_pack_json": display_path(blind_pack_json), "blind_pack_markdown": display_path(blind_pack_md), "decisions": display_path(decisions_path), @@ -287,6 +452,9 @@ def prepare_output_review_kit( output_md.parent.mkdir(parents=True, exist_ok=True) output_json.write_text(json.dumps(payload, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") output_md.write_text(render_markdown(payload), encoding="utf-8") + if output_html: + output_html.parent.mkdir(parents=True, exist_ok=True) + output_html.write_text(render_html(payload), encoding="utf-8") return payload @@ -297,6 +465,7 @@ def main() -> None: parser.add_argument("--decisions", default=str(DEFAULT_DECISIONS)) parser.add_argument("--output-json", default=str(DEFAULT_OUTPUT_JSON)) parser.add_argument("--output-md", default=str(DEFAULT_OUTPUT_MD)) + parser.add_argument("--output-html", default=str(DEFAULT_OUTPUT_HTML)) parser.add_argument("--write-template", action="store_true") args = parser.parse_args() @@ -306,6 +475,7 @@ def main() -> None: Path(args.decisions).resolve(), Path(args.output_json).resolve(), Path(args.output_md).resolve(), + Path(args.output_html).resolve() if args.output_html else None, write_template=args.write_template, ) print(json.dumps(payload, ensure_ascii=False, indent=2)) diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index 9b63029f..12cc1a33 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -142,13 +142,13 @@ ACTION_GUIDANCE: dict[str, dict[str, str]] = { "output-lab": { "summary": "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。", "why": "没有输出质量和人工盲评证据时，Skill 只能证明会触发，不能证明输出真的更好且经得起审查。", - "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_decisions.json + reports/output_review_adjudication.md", + "source_fix": "evals/output/cases.jsonl + reports/output_quality_scorecard.md + reports/output_review_kit.html + reports/output_review_adjudication.md", "source_paths": [ {"path": "evals/output/cases.jsonl", "label": "output eval cases", "kind": "eval", "patterns": ["case_id"]}, {"path": "reports/output_quality_scorecard.md", "label": "output scorecard", "kind": "report", "patterns": ["# Output"]}, {"path": "reports/output_execution_runs.md", "label": "output execution runs", "kind": "report", "patterns": ["# Output Execution"]}, {"path": "reports/output_blind_review_pack.md", "label": "blind A/B review pack", "kind": "report", "patterns": ["# Output Blind"]}, - {"path": "reports/output_review_decisions.json", "label": "review decisions template", "kind": "report", "patterns": ["decision_contract", "winner_variant"]}, + {"path": "reports/output_review_kit.html", "label": "reviewer cockpit", "kind": "report", "patterns": ["Output Review Kit", "Variant A"]}, {"path": "reports/output_review_adjudication.md", "label": "review adjudication", "kind": "report", "patterns": ["# Output Review"]}, ], "verification": "python3 scripts/adjudicate_output_review.py --write-template && python3 scripts/yao.py output-review", @@ -736,7 +736,7 @@ def render_html(report: dict[str, Any]) -> str:
    -

    评审方式

    先看 reports/output_blind_review_pack.md 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

    +

    评审方式

    先打开 reports/output_review_kit.html 做盲评，填入 reports/output_review_decisions.json，再用 reports/output_review_adjudication.md 核对答案 key。

    运行方式

    reports/output_execution_runs.md 会标明 recorded fixture、command run 或 model run；只有 provider runner 返回 model metadata 时才算 model-executed。

    diff --git a/scripts/review_studio_data.py b/scripts/review_studio_data.py index 5e77b671..f9407b34 100644 --- a/scripts/review_studio_data.py +++ b/scripts/review_studio_data.py @@ -64,6 +64,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "output_execution": "reports/output_execution_runs.md", "output_blind_review": "reports/output_blind_review_pack.md", "output_review_kit": "reports/output_review_kit.md", + "output_review_kit_html": "reports/output_review_kit.html", "output_review_decisions": "reports/output_review_decisions.json", "output_review_adjudication": "reports/output_review_adjudication.md", "benchmark_reproducibility": "reports/benchmark_reproducibility.md", diff --git a/scripts/yao.py b/scripts/yao.py index a9261d66..8559ce6f 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -203,6 +203,7 @@ def command_report(args: argparse.Namespace) -> int: "compiled_targets": "reports/compiled_targets.json", "output_execution": "reports/output_execution_runs.json", "output_review_kit": "reports/output_review_kit.json", + "output_review_kit_html": "reports/output_review_kit.html", "output_review_adjudication": "reports/output_review_adjudication.json", "adoption_drift": "reports/adoption_drift_report.json", "telemetry_hooks": "reports/telemetry_hook_recipes.json", @@ -515,6 +516,8 @@ def command_output_review_kit(args: argparse.Namespace) -> int: cmd.extend(["--output-json", args.output_json]) if args.output_md: cmd.extend(["--output-md", args.output_md]) + if args.output_html: + cmd.extend(["--output-html", args.output_html]) if args.write_template: cmd.append("--write-template") result = run_script("prepare_output_review_kit.py", cmd) diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index 346edd65..96b011c9 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -596,6 +596,7 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] output_review_kit_cmd.add_argument("--decisions") output_review_kit_cmd.add_argument("--output-json") output_review_kit_cmd.add_argument("--output-md") + output_review_kit_cmd.add_argument("--output-html") output_review_kit_cmd.add_argument("--write-template", action="store_true") output_review_kit_cmd.set_defaults(func=_handler(command_handlers, "command_output_review_kit")) diff --git a/tests/verify_output_review_kit.py b/tests/verify_output_review_kit.py index ae6e265d..dbd61aca 100644 --- a/tests/verify_output_review_kit.py +++ b/tests/verify_output_review_kit.py @@ -67,6 +67,7 @@ def main() -> None: decisions_path = tmp_root / "output_review_decisions.json" kit_json = tmp_root / "output_review_kit.json" kit_md = tmp_root / "output_review_kit.md" + kit_html = tmp_root / "output_review_kit.html" pending_proc = run( [ str(REVIEW_KIT), @@ -80,6 +81,8 @@ def main() -> None: str(kit_json), "--output-md", str(kit_md), + "--output-html", + str(kit_html), "--write-template", ] ) @@ -93,6 +96,7 @@ def main() -> None: assert pending_payload["summary"]["answer_key_path_exposed"] is False, pending_payload assert pending_payload["summary"]["ready_to_run_adjudication"] is False, pending_payload assert "answer_key" not in pending_payload["artifacts"], pending_payload["artifacts"] + assert pending_payload["artifacts"]["reviewer_kit_html"].endswith("tests/tmp_output_review_kit/output_review_kit.html"), pending_payload["artifacts"] assert len(pending_payload["cases"]) == 5, pending_payload["cases"] first_case = pending_payload["cases"][0] assert first_case["variant_a"]["output"], first_case @@ -102,12 +106,18 @@ def main() -> None: kit_json_text = kit_json.read_text(encoding="utf-8") kit_md_text = kit_md.read_text(encoding="utf-8") + kit_html_text = kit_html.read_text(encoding="utf-8") assert "# Output Review Kit" in kit_md_text, kit_md_text assert "answer key path exposed: `false`" in kit_md_text, kit_md_text assert "Variant A" in kit_md_text and "Variant B" in kit_md_text, kit_md_text assert "python3 scripts/yao.py output-review" in kit_md_text, kit_md_text + assert "Output Review Kit" in kit_html_text, kit_html_text + assert "Reviewer cockpit for output quality decisions" in kit_html_text, kit_html_text + assert "Variant A" in kit_html_text and "Variant B" in kit_html_text, kit_html_text + assert "Decision Template" in kit_html_text, kit_html_text assert_no_answer_key_leak(kit_json_text) assert_no_answer_key_leak(kit_md_text) + assert_no_answer_key_leak(kit_html_text) cli_proc = run( [ @@ -123,6 +133,8 @@ def main() -> None: str(tmp_root / "cli_output_review_kit.json"), "--output-md", str(tmp_root / "cli_output_review_kit.md"), + "--output-html", + str(tmp_root / "cli_output_review_kit.html"), ] ) cli_payload = json.loads(cli_proc.stdout) @@ -130,6 +142,7 @@ def main() -> None: assert cli_payload["summary"]["case_count"] == 5, cli_payload assert cli_payload["summary"]["answer_key_hidden"] is True, cli_payload assert "answer_key" not in cli_payload["artifacts"], cli_payload["artifacts"] + assert (tmp_root / "cli_output_review_kit.html").exists(), cli_payload template = json.loads(decisions_path.read_text(encoding="utf-8")) filled = { @@ -160,6 +173,8 @@ def main() -> None: str(tmp_root / "ready_output_review_kit.json"), "--output-md", str(tmp_root / "ready_output_review_kit.md"), + "--output-html", + str(tmp_root / "ready_output_review_kit.html"), ] ) ready_payload = json.loads(ready_proc.stdout) @@ -188,6 +203,8 @@ def main() -> None: str(tmp_root / "invalid_output_review_kit.json"), "--output-md", str(tmp_root / "invalid_output_review_kit.md"), + "--output-html", + str(tmp_root / "invalid_output_review_kit.html"), ] ) invalid_payload = json.loads(invalid_proc.stdout) diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index f7fde2de..95a0c83d 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -21,6 +21,7 @@ def main() -> None: shutil.rmtree(tmp_root) tmp_root.mkdir(parents=True, exist_ok=True) subprocess.run([sys.executable, str(ROOT / "scripts" / "run_output_eval.py")], cwd=ROOT, check=True, capture_output=True, text=True) + subprocess.run([sys.executable, str(ROOT / "scripts" / "prepare_output_review_kit.py")], cwd=ROOT, check=True, capture_output=True, text=True) subprocess.run( [ sys.executable, @@ -360,6 +361,7 @@ def main() -> None: assert full_payload["evidence_paths"]["compiled_targets"] == "reports/compiled_targets.md", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_execution"] == "reports/output_execution_runs.md", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_blind_review"] == "reports/output_blind_review_pack.md", full_payload["evidence_paths"] + assert full_payload["evidence_paths"]["output_review_kit_html"] == "reports/output_review_kit.html", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_review_decisions"] == "reports/output_review_decisions.json", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_review_adjudication"] == "reports/output_review_adjudication.md", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["python_compatibility"] == "reports/python_compatibility.md", full_payload["evidence_paths"] @@ -479,7 +481,7 @@ def main() -> None: "reports/output_quality_scorecard.md", "reports/output_execution_runs.md", "reports/output_blind_review_pack.md", - "reports/output_review_decisions.json", + "reports/output_review_kit.html", "reports/output_review_adjudication.md", }, synthetic_actions assert all(item["exists"] for item in synthetic_actions[0]["source_refs"]), synthetic_actions @@ -494,7 +496,7 @@ def main() -> None: assert "审查闸门" in html, html[:1200] assert "修复动作" in html, html[:3000] assert "补足 output eval 覆盖、execution evidence、blind A/B 和 reviewer adjudication。" in html, html[:9000] - assert "reports/output_review_decisions.json" in html, html[:9000] + assert "reports/output_review_kit.html" in html, html[:9000] assert "python3 scripts/adjudicate_output_review.py --write-template" in html, html[:9000] assert "对保留的 warning 写入 reviewer、理由、范围和到期时间，或修掉 warning。" in html, html[:9000] assert "补齐 provider、真人盲评、原生权限执行和真实客户端遥测证据" in html, html
    路径Path作用Role类型Type
    SKILL.mdSkill 入口文件Skill entrypoint文件file
    README.md人类可读使用说明Human-readable usage guide文件file
    agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
    manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
    references扩展指导与复用资料Extended guidance and reusable notes目录folder
    scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
    evals触发与质量检查Trigger and quality checks目录folder
    reports生成的证据与总结报告Generated evidence and overview artifacts目录folder