frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

feat: add world class evidence ledger

Browse Source
This commit is contained in:
yaojingang
2026-06-13 23:42:31 +08:00
parent be3fe00161
commit f3b23ea9b9
47 changed files with 1774 additions and 326 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+1
View File
@@ -68,6 +68,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept
- `scripts/review_studio_gates.py`: Review Studio gate evaluation, release decision scoring, and gate status labels.
- `scripts/render_skill_os2_audit.py`: requirement-by-requirement Skill OS 2.0 completion audit. Keep local evidence, human-required gaps, and external-required gaps separate so reports do not overclaim world-class readiness.
- `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete.
- `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence.
- `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable.
- `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation.
- `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers.
Makefile
+6 -3
View File
@@ -2,7 +2,7 @@ PYTHON ?= python3
LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill
ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check world-class-ledger-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean
eval:
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
@@ -91,6 +91,9 @@ skill-os2-audit-check:
world-class-evidence-check:
$(PYTHON) tests/verify_world_class_evidence_plan.py
world-class-ledger-check:
$(PYTHON) tests/verify_world_class_evidence_ledger.py
benchmark-reproducibility-check:
$(PYTHON) tests/verify_benchmark_reproducibility.py
@@ -208,11 +211,11 @@ sync-local-install: package-check
sync-active-install: package-check
$(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)"
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check world-class-ledger-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check
ci-test:
$(PYTHON) scripts/ci_test.py
clean:
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_world_class_evidence tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md
find . -type d -name __pycache__ -prune -exec rm -rf {} +
README.md
+5 -2
View File
@@ -25,6 +25,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
- a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, adoption drift, reviewer waivers, reviewer annotations, release evidence, and per-warning fix actions
- a Skill OS 2.0 audit that maps each world-class requirement to current evidence, human-required gaps, and external-required gaps
- a world-class evidence plan that turns remaining provider, human, native-permission, and real-client telemetry gaps into executable evidence tasks
- a world-class evidence ledger that records which external and human evidence is accepted or still pending without treating planned work as proof
- a benchmark reproducibility manifest that checks methodology sections, required artifacts, failure disclosure, and reproduction commands
- Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports
- a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks
@@ -269,7 +270,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa
- root governance maturity score: `90/100`; governed benchmark example: `95/100`
- CJK-aware trigger matching is now covered by explicit Chinese build, packaging, eval, and near-neighbor cases
- context budgets: root `944/1000`, complex benchmark `790/1000`, governed benchmark `760/1000`
- quality density: root `131.7`, complex benchmark `164.6`, governed benchmark `171.1`
- quality density: root `137.7`, complex benchmark `164.6`, governed benchmark `171.1`
- regression milestones are tracked in [reports/regression_history.md](reports/regression_history.md)
- description drift history is tracked in [reports/description_drift_history.md](reports/description_drift_history.md)
- route confusion is tracked in [reports/route_scorecard.md](reports/route_scorecard.md)
@@ -281,6 +282,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa
- Review Studio fix actions are embedded in [reports/review-studio.json](reports/review-studio.json)
- reviewer waiver evidence is published in [reports/review_waivers.md](reports/review_waivers.md)
- remaining world-class evidence tasks are published in [reports/world_class_evidence_plan.md](reports/world_class_evidence_plan.md)
- current world-class evidence acceptance state is published in [reports/world_class_evidence_ledger.md](reports/world_class_evidence_ledger.md)
- benchmark reproducibility evidence is published in [reports/benchmark_reproducibility.md](reports/benchmark_reproducibility.md)
- target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md)
- registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md)
@@ -397,7 +399,7 @@ Utility scripts that make the meta-skill operational:
- `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries
- `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs
- `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence
- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, world-class-evidence, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint
- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, world-class-evidence, world-class-ledger, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint
- `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report
- `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot
- `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger
@@ -408,6 +410,7 @@ Utility scripts that make the meta-skill operational:
- `render_regression_history.py`: turns milestone snapshots into a readable regression history report
- `render_skill_os2_audit.py`: renders a requirement-by-requirement Skill OS 2.0 audit that separates landed local evidence from human-required and external-required gaps
- `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence
- `render_world_class_evidence_ledger.py`: renders a machine-checkable ledger for current world-class evidence acceptance, anti-overclaim guards, provenance requirements, and privacy contracts
- `render_benchmark_reproducibility.py`: renders methodology, artifact, failure-disclosure, and reproduction-command evidence for public benchmark claims
- `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation
- `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage
registry/index.json
+1 -1
View File
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c"
"package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f"
}
]
}
registry/packages/yao-meta-skill.json
+3 -3
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c",
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a"
"package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f",
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328"
},
"compatibility": {
"openai": "pass",
@@ -48,7 +48,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a",
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
reports/benchmark_reproducibility.json
+30 -17
View File
@@ -3,19 +3,19 @@
"ok": true,
"generated_at": "2026-06-13",
"skill_dir": ".",
"commit": "c2507af5267e3c1eb6802449f74236c5ab649742",
"commit": "be3fe0016116ab337a972a0bfee1b05cace7ac36",
"git_status": {
"available": true,
"dirty": true,
"changed_file_count": 43,
"changed_file_count": 47,
"sample": [
" M AGENTS.md",
" M Makefile",
" M README.md",
" M registry/index.json",
" M registry/packages/yao-meta-skill.json",
" M reports/adoption_drift_report.json",
" M reports/benchmark_methodology.md",
" M reports/benchmark_reproducibility.json",
" M reports/benchmark_reproducibility.md",
" M reports/compiled_targets.json",
" M reports/context_budget.json",
" M reports/install_simulation.json",
@@ -26,11 +26,11 @@
"summary": {
"reproducibility_ready": true,
"methodology_complete": true,
"required_artifact_count": 16,
"required_artifact_count": 17,
"missing_artifact_count": 0,
"output_case_count": 5,
"failure_disclosure_count": 3,
"command_count": 15,
"command_count": 16,
"command_executed_count": 10,
"timing_observed_count": 10,
"model_executed_count": 0,
@@ -40,8 +40,9 @@
"world_class_ready": false,
"world_class_open_gap_count": 4,
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 43
"changed_file_count": 47
},
"methodology": {
"path": "reports/benchmark_methodology.md",
@@ -114,8 +115,8 @@
"label": "output_execution",
"path": "reports/output_execution_runs.json",
"exists": true,
"bytes": 7967,
"sha256": "b3fecad2a0c31f48238a2be03e10a870d4e9b83c9af3bdfaba4f14514e44eb71"
"bytes": 7966,
"sha256": "13838e344e1c707fea704bed95d80d499fea8e673a4a0fdc519b2671a625a4d1"
},
{
"label": "blind_review",
@@ -149,43 +150,50 @@
"label": "trust_report",
"path": "reports/security_trust_report.json",
"exists": true,
"bytes": 81801,
"sha256": "b3513361608feadd78ceeda13b047aa6aa933ba6f8f94687e92831843e710075"
"bytes": 82914,
"sha256": "f709657c5ddb65011675d83f9170240722508831c0ac60ab37b57d87d8707496"
},
{
"label": "registry_audit",
"path": "reports/registry_audit.json",
"exists": true,
"bytes": 3183,
"sha256": "f1ec4e2f7e8c0bab0c9e2fd5c1007d9cd368f5d4a2a05c94dbc1bce12cb67e63"
"sha256": "3966d27f6a71b2f518b9b8e74d2853ffdbc778f84669d81012f671b41dcc0544"
},
{
"label": "package_verification",
"path": "reports/package_verification.json",
"exists": true,
"bytes": 19325,
"sha256": "3183e66db7e31d4188dac8bab9f42e6fe52b5e98fdb8009655caa8468d18c9d9"
"sha256": "0908995b45fb0f226a8b21f61c096ef432c95c267c7b4614efbc940edc74db8a"
},
{
"label": "install_simulation",
"path": "reports/install_simulation.json",
"exists": true,
"bytes": 8557,
"sha256": "0befebd76310b0f02d131814003ff2c918802f5f6c1f356c139479ed1c1b87c6"
"sha256": "db509b728ad4d9844f146f2f7493b9180bf34ef2c75e7e2d7e57868f3b612026"
},
{
"label": "skill_os2_audit",
"path": "reports/skill_os2_audit.json",
"exists": true,
"bytes": 13946,
"sha256": "7e17b3a890f92d7268b26ed15035ed191d1b478b7f96029628c9168a0a09dc33"
"sha256": "a4f4e6e665e4950289df34f6ef3ae04000666d6588fe05ca333c0809ad1acbda"
},
{
"label": "world_class_evidence_plan",
"path": "reports/world_class_evidence_plan.json",
"exists": true,
"bytes": 7555,
"sha256": "f8b28a7b02b738153ce25e4a1fc47c2b3c8236cf26ff1d53e6425816a3e48a78"
"bytes": 7614,
"sha256": "c24ea93bc38e7c5830679ab4e3a497ec06734aed274d3f8cf98befd7a31cea0e"
},
{
"label": "world_class_evidence_ledger",
"path": "reports/world_class_evidence_ledger.json",
"exists": true,
"bytes": 8269,
"sha256": "3129158ca088b4e5fc99107f35a6a4abed3080b9eb87eb992a8c29711475394d"
}
],
"missing_artifacts": [],
@@ -260,6 +268,11 @@
"command": "python3 scripts/yao.py world-class-evidence .",
"evidence": "reports/world_class_evidence_plan.json"
},
{
"label": "world-class evidence ledger",
"command": "python3 scripts/yao.py world-class-ledger .",
"evidence": "reports/world_class_evidence_ledger.json"
},
{
"label": "full ci",
"command": "make ci-test",
reports/benchmark_reproducibility.md
+14 -11
View File
@@ -1,22 +1,22 @@
# Benchmark Reproducibility
Generated at: `2026-06-13`
Commit: `c2507af5267e3c1eb6802449f74236c5ab649742`
Commit: `be3fe0016116ab337a972a0bfee1b05cace7ac36`
Working tree dirty at generation: `true`
## Summary
- reproducibility ready: `true`
- methodology complete: `true`
- required artifacts: `16`
- required artifacts: `17`
- missing artifacts: `0`
- output cases: `5`
- disclosed failure cases: `3`
- reproduction commands: `15`
- reproduction commands: `16`
- provider evidence complete: `false`
- human review complete: `false`
- world-class ready: `false`
- changed files at generation: `43`
- changed files at generation: `47`
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.
@@ -40,17 +40,18 @@ This report proves local benchmark reproducibility only. It keeps external provi
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
| output_execution | `reports/output_execution_runs.json` | present | `b3fecad2a0c3` |
| output_execution | `reports/output_execution_runs.json` | present | `13838e344e1c` |
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
| review_adjudication | `reports/output_review_adjudication.json` | present | `ddd9af90d42e` |
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` |
| trust_report | `reports/security_trust_report.json` | present | `b3513361608f` |
| registry_audit | `reports/registry_audit.json` | present | `f1ec4e2f7e8c` |
| package_verification | `reports/package_verification.json` | present | `3183e66db7e3` |
| install_simulation | `reports/install_simulation.json` | present | `0befebd76310` |
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `7e17b3a890f9` |
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `f8b28a7b02b7` |
| trust_report | `reports/security_trust_report.json` | present | `f709657c5ddb` |
| registry_audit | `reports/registry_audit.json` | present | `3966d27f6a71` |
| package_verification | `reports/package_verification.json` | present | `0908995b45fb` |
| install_simulation | `reports/install_simulation.json` | present | `db509b728ad4` |
| skill_os2_audit | `reports/skill_os2_audit.json` | present | `a4f4e6e665e4` |
| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `c24ea93bc38e` |
| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `3129158ca088` |
## Reproduction Commands
@@ -82,6 +83,8 @@ This report proves local benchmark reproducibility only. It keeps external provi
- evidence: `reports/skill_os2_audit.json`
- `python3 scripts/yao.py world-class-evidence .`
- evidence: `reports/world_class_evidence_plan.json`
- `python3 scripts/yao.py world-class-ledger .`
- evidence: `reports/world_class_evidence_ledger.json`
- `make ci-test`
- evidence: `CI target output`
reports/compiled_targets.json
+80 -65
View File
@@ -196,6 +196,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
@@ -274,7 +275,7 @@
],
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
}
@@ -380,7 +381,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -425,6 +426,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -473,14 +475,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -502,7 +504,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -523,7 +525,7 @@
"strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -534,7 +536,7 @@
},
"scripts": {
"strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -615,7 +617,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -660,6 +662,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -708,14 +711,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -735,7 +738,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -756,7 +759,7 @@
"strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -767,7 +770,7 @@
},
"scripts": {
"strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -1020,6 +1023,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
@@ -1098,7 +1102,7 @@
],
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
}
@@ -1204,7 +1208,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -1249,6 +1253,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -1297,14 +1302,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -1326,7 +1331,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -1347,7 +1352,7 @@
"strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -1358,7 +1363,7 @@
},
"scripts": {
"strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -1439,7 +1444,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -1484,6 +1489,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -1532,14 +1538,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -1559,7 +1565,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -1580,7 +1586,7 @@
"strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -1591,7 +1597,7 @@
},
"scripts": {
"strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -1844,6 +1850,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
@@ -1922,7 +1929,7 @@
],
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
}
@@ -2028,7 +2035,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -2073,6 +2080,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -2121,14 +2129,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -2150,7 +2158,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -2171,7 +2179,7 @@
"strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -2181,7 +2189,7 @@
},
"scripts": {
"strategy": "Expose script and permission metadata for downstream clients or installers to enforce.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -2256,7 +2264,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -2301,6 +2309,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -2349,14 +2358,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -2376,7 +2385,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -2397,7 +2406,7 @@
"strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -2407,7 +2416,7 @@
},
"scripts": {
"strategy": "Expose script and permission metadata for downstream clients or installers to enforce.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -2652,6 +2661,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
@@ -2730,7 +2740,7 @@
],
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
}
@@ -2836,7 +2846,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -2881,6 +2891,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -2929,14 +2940,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -2958,7 +2969,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -2979,7 +2990,7 @@
"strategy": "Keep optional directories as relative resources next to SKILL.md.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -2990,7 +3001,7 @@
},
"scripts": {
"strategy": "Scripts remain local optional resources and should advertise --help when executable.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -3064,7 +3075,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -3109,6 +3120,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3157,14 +3169,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3184,7 +3196,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -3205,7 +3217,7 @@
"strategy": "Keep optional directories as relative resources next to SKILL.md.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -3216,7 +3228,7 @@
},
"scripts": {
"strategy": "Scripts remain local optional resources and should advertise --help when executable.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -3460,6 +3472,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
@@ -3538,7 +3551,7 @@
],
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
}
@@ -3644,7 +3657,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -3689,6 +3702,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3737,14 +3751,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3766,7 +3780,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -3787,7 +3801,7 @@
"strategy": "Install as project or user scoped skill source, preserving relative references and scripts.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -3798,7 +3812,7 @@
},
"scripts": {
"strategy": "Scripts require workspace trust and operator/client approval outside this compiler.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
@@ -3876,7 +3890,7 @@
},
"file_write": {
"required": true,
"script_count": 54,
"script_count": 55,
"scripts": [
"scripts/adjudicate_output_review.py",
"scripts/build_confusion_matrix.py",
@@ -3921,6 +3935,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
@@ -3969,14 +3984,14 @@
},
"help_smoke": {
"enabled": true,
"checked_count": 67,
"checked_count": 68,
"failed_count": 0,
"failed_scripts": []
},
"trust_summary": {
"secret_findings": 0,
"network_script_count": 3,
"file_write_script_count": 54,
"file_write_script_count": 55,
"subprocess_script_count": 8,
"interactive_script_count": 0,
"help_smoke_failed_count": 0
@@ -3996,7 +4011,7 @@
],
"capability_counts": {
"network": 3,
"file_write": 54,
"file_write": 55,
"subprocess": 8,
"interactive": 0
},
@@ -4017,7 +4032,7 @@
"strategy": "Install as project or user scoped skill source, preserving relative references and scripts.",
"counts": {
"references": 32,
"scripts": 77,
"scripts": 78,
"assets": 2,
"reports": 43
},
@@ -4028,7 +4043,7 @@
},
"scripts": {
"strategy": "Scripts require workspace trust and operator/client approval outside this compiler.",
"script_count": 77,
"script_count": 78,
"help_smoke_failed_count": 0
},
"permissions": {
reports/context_budget.json
+3 -3
View File
@@ -6,10 +6,10 @@
"context_budget_tier": "production",
"context_budget_limit": 1000,
"skill_body_tokens": 751,
"other_text_tokens": 985553,
"other_text_tokens": 1002467,
"estimated_initial_load_tokens": 944,
"estimated_total_text_tokens": 986304,
"relevant_file_count": 431,
"estimated_total_text_tokens": 1003218,
"relevant_file_count": 436,
"unused_resource_dirs": [],
"quality_signal_points": 130,
"quality_density": 137.7
reports/install_simulation.json
+1 -1
View File
@@ -8,7 +8,7 @@
"installed_skill_dir": "[temporary-install-root]/yao-meta-skill",
"summary": {
"archive_present": true,
"archive_entry_count": 517,
"archive_entry_count": 521,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.11,
"duration_ms": 28.6,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 27.48,
"duration_ms": 28.28,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.22,
"duration_ms": 28.23,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.36,
"duration_ms": 28.19,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.99,
"duration_ms": 28.68,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.13,
"duration_ms": 28.96,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.35,
"duration_ms": 28.39,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.95,
"duration_ms": 28.41,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 26.04,
"duration_ms": 28.45,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 25.98,
"duration_ms": 28.25,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 26.11 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.48 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 26.22 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.36 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 25.99 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.13 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.35 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.95 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.04 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.98 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 28.6 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 28.28 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 28.23 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.19 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.68 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.96 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.39 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.41 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.45 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.25 | 98 | 100.0 | pass |
## Next Fixes
reports/package_verification.json
+2 -2
View File
@@ -8,8 +8,8 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a",
"archive_entry_count": 517,
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328",
"archive_entry_count": 521,
"failure_count": 0,
"warning_count": 0
},
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a`
- Archive SHA256: `23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328`
- Failures: `0`
- Warnings: `0`
reports/registry_audit.json
+4 -4
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c",
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a"
"package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f",
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328"
},
"compatibility": {
"openai": "pass",
@@ -53,7 +53,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a",
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c"
"package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c`
- Archive SHA256: `4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a`
- Package SHA256: `22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f`
- Archive SHA256: `23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328`
- Install simulated: `True`
## Compatibility
reports/review-studio.html
+6 -6
View File
@@ -242,12 +242,12 @@
<section>
<h2>核心指标</h2>
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native enforcement targets</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>77 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>517 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
<div class="metrics"><article class='metric'><span>Skill IR</span><strong>2.0.0</strong><p>5 targets in platform-neutral contract</p></article><article class='metric'><span>Compiler</span><strong>5/5</strong><p>target contracts compiled from Skill IR</p></article><article class='metric'><span>Output Delta</span><strong>100.0</strong><p>5 cases; 1 file-backed</p></article><article class='metric'><span>Exec Runs</span><strong>10</strong><p>command 10; model 0; recorded 0</p></article><article class='metric'><span>Blind A/B</span><strong>5</strong><p>review pairs hide baseline vs with-skill labels</p></article><article class='metric'><span>Review A/B</span><strong>0/5</strong><p>adjudication decisions; pending 5</p></article><article class='metric'><span>Runtime</span><strong>5/5</strong><p>target conformance pass rate</p></article><article class='metric'><span>Perm Probe</span><strong>4/4</strong><p>0 native enforcement targets</p></article><article class='metric'><span>Trust</span><strong>0</strong><p>78 scripts scanned; secrets found</p></article><article class='metric'><span>Atlas</span><strong>5</strong><p>12 scanned skills; route collisions</p></article><article class='metric'><span>Drift</span><strong>low</strong><p>1 metadata events; 0 missed triggers</p></article><article class='metric'><span>Waivers</span><strong>0</strong><p>0 gates covered; human risk decisions</p></article><article class='metric'><span>Notes</span><strong>0/0</strong><p>0 open blocker annotations</p></article><article class='metric'><span>Registry</span><strong>1.1.0</strong><p>5 targets; MIT license</p></article><article class='metric'><span>Archive</span><strong>pass</strong><p>521 zip entries; package verification</p></article><article class='metric'><span>Install</span><strong>pass</strong><p>4 adapters; 12 permissions enforced; 0 permission failures</p></article><article class='metric'><span>Upgrade</span><strong>minor</strong><p>declared minor; 0 breaking changes</p></article></div>
</section>
<section>
<h2>审查闸门</h2>
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 77 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
<div class="gates"><article class='gate pass'><div><span>通过</span><h3>意图画布</h3></div><p>intent confidence 100/100; Intent is clear enough to package the first routeable version.</p><footer>reports/intent-confidence.json <a href='intent-confidence.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>触发实验</h3></div><p>13 trigger cases; 0 misroutes; 0 ambiguous</p><footer>reports/route_scorecard.json <a href='route_scorecard.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>输出实验</h3></div><p>5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5</p><footer>reports/output_quality_scorecard.json <a href='output_quality_scorecard.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>上下文</h3></div><p>initial load 944/1000; quality density 137.7</p><footer>reports/context_budget.json <a href='context_budget.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运行矩阵</h3></div><p>5 / 5 targets pass</p><footer>reports/conformance_matrix.json <a href='conformance_matrix.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>信任报告</h3></div><p>0 secrets; 78 scripts; 3 network-capable scripts; 0 help smoke failures</p><footer>reports/security_trust_report.json <a href='security_trust_report.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限批准</h3></div><p>3/3 permissions approved; gaps 0; required file_write, network, subprocess</p><footer>reports/security_trust_report.json + security/permission_policy.json <a href='../security/permission_policy.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>权限探针</h3></div><p>4/4 targets probed; native 0; metadata fallback 4; residual risks 4</p><footer>reports/runtime_permission_probes.json <a href='runtime_permission_probes.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>组合治理</h3></div><p>12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues</p><footer>reports/skill_atlas.json <a href='skill_atlas.html'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>运营回路</h3></div><p>1 metadata events; adoption 0; missed 0; bad-output 0; risk low</p><footer>reports/adoption_drift_report.json <a href='adoption_drift_report.md'>证据</a></footer></article><article class='gate warn'><div><span>关注</span><h3>人工批准</h3></div><p>0 active waivers; 1 warning gates still need reviewer decision</p><footer>reports/review_waivers.json <a href='review_waivers.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>注册审计</h3></div><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p><footer>reports/registry_audit.json + reports/install_simulation.json <a href='registry_audit.md'>证据</a></footer></article><article class='gate pass'><div><span>通过</span><h3>发布路线</h3></div><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p><footer>reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md <a href='promotion_decisions.md'>证据</a></footer></article></div>
</section>
<div class="twocol">
@@ -273,7 +273,7 @@
</div>
<div class="panel">
<h2>证据路径</h2>
<ul class="evidence"><li><strong>skill_overview</strong><span>reports/skill-overview.html</span></li><li><strong>review_viewer</strong><span>reports/review-viewer.html</span></li><li><strong>output_eval</strong><span>reports/output_quality_scorecard.md</span></li><li><strong>output_execution</strong><span>reports/output_execution_runs.md</span></li><li><strong>output_blind_review</strong><span>reports/output_blind_review_pack.md</span></li><li><strong>output_review_decisions</strong><span>reports/output_review_decisions.json</span></li><li><strong>output_review_adjudication</strong><span>reports/output_review_adjudication.md</span></li><li><strong>benchmark_reproducibility</strong><span>reports/benchmark_reproducibility.md</span></li><li><strong>runtime_conformance</strong><span>reports/conformance_matrix.md</span></li><li><strong>trust_report</strong><span>reports/security_trust_report.md</span></li><li><strong>permission_policy</strong><span>security/permission_policy.md</span></li><li><strong>runtime_permissions</strong><span>reports/runtime_permission_probes.md</span></li><li><strong>skill_atlas</strong><span>reports/skill_atlas.html</span></li><li><strong>compiled_targets</strong><span>reports/compiled_targets.md</span></li><li><strong>adoption_drift</strong><span>reports/adoption_drift_report.md</span></li><li><strong>review_waivers</strong><span>reports/review_waivers.md</span></li><li><strong>review_annotations</strong><span>reports/review_annotations.md</span></li><li><strong>world_class_evidence_plan</strong><span>reports/world_class_evidence_plan.md</span></li><li><strong>registry_audit</strong><span>reports/registry_audit.md</span></li><li><strong>package_verification</strong><span>reports/package_verification.md</span></li><li><strong>install_simulation</strong><span>reports/install_simulation.md</span></li><li><strong>upgrade_check</strong><span>reports/upgrade_check.md</span></li><li><strong>migration</strong><span>docs/migration-v2.md</span></li></ul>
<ul class="evidence"><li><strong>skill_overview</strong><span>reports/skill-overview.html</span></li><li><strong>review_viewer</strong><span>reports/review-viewer.html</span></li><li><strong>output_eval</strong><span>reports/output_quality_scorecard.md</span></li><li><strong>output_execution</strong><span>reports/output_execution_runs.md</span></li><li><strong>output_blind_review</strong><span>reports/output_blind_review_pack.md</span></li><li><strong>output_review_decisions</strong><span>reports/output_review_decisions.json</span></li><li><strong>output_review_adjudication</strong><span>reports/output_review_adjudication.md</span></li><li><strong>benchmark_reproducibility</strong><span>reports/benchmark_reproducibility.md</span></li><li><strong>runtime_conformance</strong><span>reports/conformance_matrix.md</span></li><li><strong>trust_report</strong><span>reports/security_trust_report.md</span></li><li><strong>permission_policy</strong><span>security/permission_policy.md</span></li><li><strong>runtime_permissions</strong><span>reports/runtime_permission_probes.md</span></li><li><strong>skill_atlas</strong><span>reports/skill_atlas.html</span></li><li><strong>compiled_targets</strong><span>reports/compiled_targets.md</span></li><li><strong>adoption_drift</strong><span>reports/adoption_drift_report.md</span></li><li><strong>review_waivers</strong><span>reports/review_waivers.md</span></li><li><strong>review_annotations</strong><span>reports/review_annotations.md</span></li><li><strong>world_class_evidence_plan</strong><span>reports/world_class_evidence_plan.md</span></li><li><strong>world_class_evidence_ledger</strong><span>reports/world_class_evidence_ledger.md</span></li><li><strong>registry_audit</strong><span>reports/registry_audit.md</span></li><li><strong>package_verification</strong><span>reports/package_verification.md</span></li><li><strong>install_simulation</strong><span>reports/install_simulation.md</span></li><li><strong>upgrade_check</strong><span>reports/upgrade_check.md</span></li><li><strong>migration</strong><span>docs/migration-v2.md</span></li></ul>
</div>
</section>
@@ -313,7 +313,7 @@
</section>
<section id="trust" class="twocol">
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>77</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c</code></dd></div></dl></div>
<div class="panel"><h2>信任报告</h2><dl class='kv-grid'><div><dt>Secret</dt><dd>0</dd></div><div><dt>脚本数</dt><dd>78</dd></div><div><dt>网络脚本</dt><dd>3</dd></div><div><dt>Help 失败</dt><dd>0</dd></div><div><dt>包体哈希</dt><dd><code>22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f</code></dd></div></dl></div>
<div class="panel"><h2>安全边界</h2><p>高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。</p></div>
</section>
@@ -344,12 +344,12 @@
<section id="registry" class="twocol">
<div class="panel"><h2>注册审计</h2><p>yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures</p></div>
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a</code></dd></div></dl></div>
<div class="panel"><h2>包体元数据</h2><dl class='kv-grid'><div><dt>名称</dt><dd>yao-meta-skill</dd></div><div><dt>版本</dt><dd>1.1.0</dd></div><div><dt>Maturity</dt><dd>governed</dd></div><div><dt>Owner</dt><dd>Yao Team</dd></div><div><dt>License</dt><dd>MIT</dd></div><div><dt>信任级别</dt><dd>local</dd></div><div><dt>目标平台</dt><dd>openai, claude, generic, agent-skills-compatible, vscode</dd></div><div><dt>兼容通过</dt><dd>6/6</dd></div><div><dt>归档哈希</dt><dd><code>23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328</code></dd></div></dl></div>
</section>
<section id="release" class="twocol">
<div class="panel"><h2>发布路线</h2><p>0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended</p></div>
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>517</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a</code></dd></div></dl></div>
<div class="panel"><h2>包体验证</h2><dl class='kv-grid'><div><dt>目标数</dt><dd>4</dd></div><div><dt>Adapter</dt><dd>4</dd></div><div><dt>归档存在</dt><dd>是</dd></div><div><dt>Zip 条目</dt><dd>521</dd></div><div><dt>失败数</dt><dd>0</dd></div><div><dt>警告数</dt><dd>0</dd></div><div><dt>归档哈希</dt><dd><code>23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328</code></dd></div></dl></div>
</section>
<section class="twocol">
reports/review-studio.json
+568 -123
View File
File diff suppressed because it is too large Load Diff
reports/security_trust_report.json
+34 -9
View File
@@ -2,8 +2,8 @@
"ok": true,
"skill_dir": ".",
"summary": {
"scanned_files": 161,
"script_count": 77,
"scanned_files": 162,
"script_count": 78,
"internal_module_count": 10,
"secret_findings": 0,
"dependency_files": [
@@ -12,18 +12,18 @@
"network_script_count": 3,
"network_policy_covered_count": 3,
"network_policy_missing_count": 0,
"file_write_script_count": 54,
"file_write_script_count": 55,
"permission_required_count": 3,
"permission_approved_count": 3,
"permission_missing_count": 0,
"permission_invalid_count": 0,
"permission_expired_count": 0,
"help_smoke_checked_count": 67,
"help_smoke_checked_count": 68,
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 161,
"package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c"
"package_hash_file_count": 162,
"package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f"
},
"failures": [],
"warnings": [],
@@ -760,6 +760,20 @@
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/render_world_class_evidence_ledger.py",
"interface": "cli",
"interface_declared": true,
"interface_reason": "Renders a machine-checkable ledger for world-class external and human evidence gaps.",
"has_argparse": true,
"has_main_guard": true,
"uses_input": false,
"uses_network": false,
"uses_file_write": true,
"uses_subprocess": false,
"network_urls": [],
"network_hosts": []
},
{
"path": "scripts/render_world_class_evidence_plan.py",
"interface": "cli",
@@ -1155,9 +1169,9 @@
"help_smoke": {
"enabled": true,
"timeout_seconds": 5.0,
"candidate_count": 67,
"checked_count": 67,
"passed_count": 67,
"candidate_count": 68,
"checked_count": 68,
"passed_count": 68,
"failed_count": 0,
"skipped_count": 10,
"failed_scripts": [],
@@ -1672,6 +1686,16 @@
"stdout_excerpt": "usage: render_telemetry_hook_recipes.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--output-jsonl OUTPUT_JSONL]\n ",
"stderr_excerpt": ""
},
{
"path": "scripts/render_world_class_evidence_ledger.py",
"command": "python3 scripts/render_world_class_evidence_ledger.py --help",
"returncode": 0,
"timed_out": false,
"passed": true,
"has_help_text": true,
"stdout_excerpt": "usage: render_world_class_evidence_ledger.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ",
"stderr_excerpt": ""
},
{
"path": "scripts/render_world_class_evidence_plan.py",
"command": "python3 scripts/render_world_class_evidence_plan.py --help",
@@ -1971,6 +1995,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/run_conformance_suite.py",
"scripts/run_description_optimization_suite.py",
reports/security_trust_report.md
+9 -8
View File
@@ -1,22 +1,22 @@
# Security Trust Report
- OK: `True`
- Scanned files: `161`
- Scripts: `77`
- Scanned files: `162`
- Scripts: `78`
- Internal script modules: `10`
- Secret findings: `0`
- Network-capable scripts: `3`
- Network policy covered scripts: `3`
- Network policy missing scripts: `0`
- File-write scripts: `54`
- File-write scripts: `55`
- Permission approvals: `3 / 3`
- Permission approval gaps: `0`
- CLI help smoke checked: `67`
- CLI help smoke checked: `68`
- CLI help smoke failures: `0`
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `161`
- Package SHA256: `fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c`
- Package hash files: `162`
- Package SHA256: `22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f`
## Failures
@@ -54,8 +54,8 @@
- Enabled: `True`
- Timeout seconds: `5.0`
- Checked scripts: `67`
- Passed scripts: `67`
- Checked scripts: `68`
- Passed scripts: `68`
- Failed scripts: `none`
## Script Surface
@@ -113,6 +113,7 @@
| scripts/render_social_preview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_system_model.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_telemetry_hook_recipes.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. |
| scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. |
| scripts/review_studio_formatting.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to format report dictionaries as audit UI panels. |
reports/skill-os-2-review.md
+8 -5
View File
@@ -15,6 +15,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree
- Bilingual Skill Overview v2 that includes these evidence surfaces.
- Skill OS 2.0 Audit v0 for requirement-by-requirement evidence mapping, with explicit separation between local evidence, human-required review, and external-required provider/client integrations.
- World-Class Evidence Plan v0 that turns the remaining provider holdout, human adjudication, native permission enforcement, and native client telemetry gaps into executable runbooks with success checks and privacy contracts.
- World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps.
- Benchmark Reproducibility v0 that turns public benchmark methodology, required artifacts, failure disclosure, and reproduction commands into machine-checkable release evidence.
- Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review.
- Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers.
@@ -54,6 +55,7 @@ This is still not the final world-class state. Target-native behavior contracts
| Benchmark Reproducibility | `scripts/render_benchmark_reproducibility.py`, `reports/benchmark_reproducibility.md`, `tests/verify_benchmark_reproducibility.py` | v0 landed |
| Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed |
| World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed |
| World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed |
| Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed |
| Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed |
| Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed |
@@ -112,13 +114,13 @@ Next move: add real client or installer permission enforcement integration.
| Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden |
| Runtime Conformance | `5 / 5` targets passing |
| Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts |
| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `67 / 67` CLI help smoke checks passing across `77` scripts, source-contract hash scope explicit |
| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `68 / 68` CLI help smoke checks passing across `78` scripts, source-contract hash scope explicit |
| Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired |
| Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility |
| Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility |
| Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix |
| Package Verification | `4 / 4` target adapters present, archive verified, `517` zip entries, `0` failures, `0` warnings |
| Install Simulation | archive with `517` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings |
| Package Verification | `4 / 4` target adapters present, archive verified, `521` zip entries, `0` failures, `0` warnings |
| Install Simulation | archive with `521` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings |
| Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures |
| Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes |
| Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages |
@@ -127,10 +129,11 @@ Next move: add real client or installer permission enforcement integration.
| Review Studio | decision `review`, world-class score `92`, `13` gates, `0` blockers, `2` warnings, `2` review actions, `0` open annotation blockers |
| Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps |
| World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts |
| Benchmark Reproducibility | local reproducibility ready with `16` required artifacts, `0` missing artifacts, `15` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations |
| World-Class Evidence Ledger | `4` evidence entries remain pending: `1` human-required and `3` external-required; anti-overclaim guards block planned work, metadata fallbacks, pending review, and local command runners from counting as completion |
| Benchmark Reproducibility | local reproducibility ready with `17` required artifacts, `0` missing artifacts, `16` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations |
| IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable |
| Context Budget | initial load `944/1000`, under the production budget |
| CI | `make ci-test` target count is `63` after the Benchmark Reproducibility gate update |
| CI | `make ci-test` target count is `64` after the World-Class Evidence Ledger gate update |
## Next Highest-Leverage Moves
reports/skill-overview.html
+2 -2
View File
@@ -621,7 +621,7 @@
<div class="two-col">
<article class="panel">
<h3><span data-lang="zh-CN">作用定位</span><span data-lang="en">Role</span></h3>
<ul class="list"><li><span data-lang="zh-CN">从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。</span><span data-lang="en">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</span></li><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html</span><span data-lang="en">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html</span></li></ul>
<ul class="list"><li><span data-lang="zh-CN">从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。</span><span data-lang="en">Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.</span></li><li><span data-lang="zh-CN">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span><span data-lang="en">把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。</span></li><li><span data-lang="zh-CN">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html</span><span data-lang="en">交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html</span></li></ul>
</article>
<figure class="chart-figure" data-chart="flow"><svg viewBox="0 0 620 170" role="img" aria-label="交付流程"><text x="20" y="28" class="chart-title">交付流程</text><path d="M188 93 H248 M398 93 H458" class="chart-line"/><g><rect x="38" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="113" y="99" text-anchor="middle">输入材料</text></g><g><rect x="248" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="323" y="99" text-anchor="middle">Skill 包体</text></g><g><rect x="458" y="56" width="150" height="74" rx="8" fill="#F6F8FB" stroke="#e8e6dc"/><text x="533" y="99" text-anchor="middle">可复用能力</text></g></svg><figcaption>交付流程把用户输入、生成的包体和可复用能力放在一条线上。</figcaption></figure>
</div>
@@ -771,7 +771,7 @@
<p><span data-lang="zh-CN">让 reviewer 快速确认关键文件、目录和资产分布。</span><span data-lang="en">Lets reviewers confirm key files, directories, and asset distribution quickly.</span></p>
</div>
<div class="two-col">
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-0.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-1.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-2.7" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="28.3 71.7" stroke-dashoffset="-3.5" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="68.1 31.9" stroke-dashoffset="-31.9" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">320项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
<figure class="chart-figure" data-chart="asset_donut"><svg viewBox="0 0 430 270" role="img" aria-label="资产分布"><text x="20" y="30" class="chart-title">资产分布</text><circle cx="130" cy="130" r="70" fill="none" stroke="#1B365D" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="0.0" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#2D5A8A" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-0.9" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#D0DCE9" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-1.8" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#E4ECF5" stroke-width="24" stroke-dasharray="0.9 99.1" stroke-dashoffset="-2.6" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#e8e6dc" stroke-width="24" stroke-dasharray="28.1 71.9" stroke-dashoffset="-3.5" pathLength="100" transform="rotate(-90 130 130)"/><circle cx="130" cy="130" r="70" fill="none" stroke="#504e49" stroke-width="24" stroke-dasharray="68.4 31.6" stroke-dashoffset="-31.6" pathLength="100" transform="rotate(-90 130 130)"/><text x="130" y="136" text-anchor="middle">323项</text><text x="235" y="78">SKILL.md</text><text x="235" y="100">README.md</text><text x="235" y="122">agents/interface.yaml</text><text x="235" y="144">manifest.json</text><text x="235" y="166">references</text><text x="235" y="188">scripts</text></svg><figcaption>资产分布图展示当前包体的文件和目录重心。</figcaption></figure>
<table>
<thead><tr><th><span data-lang="zh-CN">路径</span><span data-lang="en">Path</span></th><th><span data-lang="zh-CN">作用</span><span data-lang="en">Role</span></th><th><span data-lang="zh-CN">类型</span><span data-lang="en">Type</span></th></tr></thead>
<tbody><tr><td>SKILL.md</td><td><span data-lang="zh-CN">Skill 入口文件</span><span data-lang="en">Skill entrypoint</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>README.md</td><td><span data-lang="zh-CN">人类可读使用说明</span><span data-lang="en">Human-readable usage guide</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>agents/interface.yaml</td><td><span data-lang="zh-CN">跨平台接口元数据</span><span data-lang="en">Neutral interface metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>manifest.json</td><td><span data-lang="zh-CN">生命周期与打包元数据</span><span data-lang="en">Lifecycle and portability metadata</span></td><td><span data-lang="zh-CN">文件</span><span data-lang="en">file</span></td></tr><tr><td>references</td><td><span data-lang="zh-CN">扩展指导与复用资料</span><span data-lang="en">Extended guidance and reusable notes</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>scripts</td><td><span data-lang="zh-CN">确定性脚本或本地工具</span><span data-lang="en">Deterministic helpers or local tooling</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>evals</td><td><span data-lang="zh-CN">触发与质量检查</span><span data-lang="en">Trigger and quality checks</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr><tr><td>reports</td><td><span data-lang="zh-CN">生成的证据与总结报告</span><span data-lang="en">Generated evidence and overview artifacts</span></td><td><span data-lang="zh-CN">目录</span><span data-lang="en">folder</span></td></tr></tbody>
reports/skill-overview.json
+225 -26
View File
@@ -30,6 +30,7 @@
"reports/adoption_drift_report.md",
"reports/review_waivers.md",
"reports/world_class_evidence_plan.md",
"reports/world_class_evidence_ledger.md",
"reports/review_annotations.md",
"reports/review-studio.html",
"reports/skill-overview.html"
@@ -435,7 +436,7 @@
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 77
"file_count": 78
},
{
"path": "evals",
@@ -447,10 +448,10 @@
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 179
"file_count": 181
}
],
"file_count": 320,
"file_count": 323,
"folder_count": 4,
"distribution": [
{
@@ -475,7 +476,7 @@
},
{
"label": "scripts",
"value": 77
"value": 78
},
{
"label": "evals",
@@ -483,7 +484,7 @@
},
{
"label": "reports",
"value": 179
"value": 181
}
]
},
@@ -604,7 +605,7 @@
"path": "scripts",
"label": "Deterministic helpers or local tooling",
"kind": "folder",
"file_count": 77
"file_count": 78
},
{
"path": "evals",
@@ -616,7 +617,7 @@
"path": "reports",
"label": "Generated evidence and overview artifacts",
"kind": "folder",
"file_count": 179
"file_count": 181
}
],
"strengths": [
@@ -888,11 +889,11 @@
"summary": {
"reproducibility_ready": true,
"methodology_complete": true,
"required_artifact_count": 16,
"required_artifact_count": 17,
"missing_artifact_count": 0,
"output_case_count": 5,
"failure_disclosure_count": 3,
"command_count": 15,
"command_count": 16,
"command_executed_count": 10,
"timing_observed_count": 10,
"model_executed_count": 0,
@@ -902,10 +903,11 @@
"world_class_ready": false,
"world_class_open_gap_count": 4,
"world_class_task_count": 4,
"world_class_ledger_pending_count": 4,
"working_tree_dirty": true,
"changed_file_count": 43
"changed_file_count": 47
},
"commit": "c2507af5267e3c1eb6802449f74236c5ab649742",
"commit": "be3fe0016116ab337a972a0bfee1b05cace7ac36",
"missing_artifacts": [],
"limitations": [
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
@@ -980,8 +982,8 @@
"failures": []
},
"trust_security": {
"scanned_files": 161,
"script_count": 77,
"scanned_files": 162,
"script_count": 78,
"internal_module_count": 10,
"secret_findings": 0,
"dependency_files": [
@@ -990,18 +992,18 @@
"network_script_count": 3,
"network_policy_covered_count": 3,
"network_policy_missing_count": 0,
"file_write_script_count": 54,
"file_write_script_count": 55,
"permission_required_count": 3,
"permission_approved_count": 3,
"permission_missing_count": 0,
"permission_invalid_count": 0,
"permission_expired_count": 0,
"help_smoke_checked_count": 67,
"help_smoke_checked_count": 68,
"help_smoke_failed_count": 0,
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 161,
"package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c"
"package_hash_file_count": 162,
"package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f"
},
"skill_atlas": {
"skill_count": 12,
@@ -1039,8 +1041,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c",
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a"
"package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f",
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328"
},
"compatibility": {
"openai": "pass",
@@ -1071,7 +1073,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a",
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -1087,8 +1089,8 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a",
"archive_entry_count": 517,
"archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328",
"archive_entry_count": 521,
"failure_count": 0,
"warning_count": 0
},
@@ -1099,7 +1101,7 @@
"ok": true,
"summary": {
"archive_present": true,
"archive_entry_count": 517,
"archive_entry_count": 521,
"archive_extracted": true,
"entrypoint_loaded": true,
"manifest_loaded": true,
@@ -1166,12 +1168,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a"
"to": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c"
"to": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f"
}
]
},
@@ -1375,7 +1377,7 @@
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.fail_count == 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
@@ -1429,6 +1431,203 @@
"open_gap_count": 4
}
},
"world_class_evidence_ledger": {
"ok": true,
"summary": {
"ledger_entry_count": 4,
"accepted_count": 0,
"pending_count": 4,
"human_pending_count": 1,
"external_pending_count": 3,
"overclaim_guard_active": true,
"ready_to_claim_world_class": false,
"decision": "evidence-pending"
},
"entries": [
{
"key": "provider-holdout",
"label": "Provider Holdout",
"category": "external",
"owner": "operator with provider credentials",
"status": "pending",
"source_status": "external_required",
"current": "model-executed 0; token-observed 0",
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
"provenance_requirements": [
"provider-backed model run",
"observed timing",
"observed token metadata"
],
"success_checks": [
"reports/output_execution_runs.json summary.model_executed_count > 0",
"reports/output_execution_runs.json summary.timing_observed_count > 0",
"reports/output_execution_runs.json summary.token_observed_count > 0",
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
],
"evidence_artifacts": [
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/skill_os2_audit.json"
],
"privacy_contract": [
"Do not commit provider credentials or environment dumps.",
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
],
"observed_state": {
"model_executed_count": 0,
"timing_observed_count": 10,
"token_observed_count": 0,
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence."
},
{
"key": "human-adjudication",
"label": "Human Adjudication",
"category": "human",
"owner": "human reviewer",
"status": "pending",
"source_status": "human_required",
"current": "0/5 decisions; pending 5",
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
"provenance_requirements": [
"real reviewer identity",
"blind A/B decisions",
"answer key unopened until decisions exist"
],
"success_checks": [
"reports/output_review_adjudication.json summary.pending_count == 0",
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
],
"evidence_artifacts": [
"reports/output_blind_review_pack.md",
"reports/output_review_decisions.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md"
],
"privacy_contract": [
"Reviewer decisions should not include raw user data or private customer detail.",
"Keep the answer key separate until after decisions are recorded."
],
"observed_state": {
"pair_count": 5,
"judgment_count": 0,
"pending_count": 5,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Record real A/B choices in the decision template, then regenerate adjudication."
},
{
"key": "native-permission-enforcement",
"label": "Native Permission Enforcement",
"category": "external",
"owner": "target client or installer integrator",
"status": "pending",
"source_status": "external_required",
"current": "native-enforced targets 0",
"objective": "Prove at least one target or installer enforces approved high-permission capabilities at runtime.",
"provenance_requirements": [
"real target or installer guard",
"native enforcement flag",
"residual risk retained for fallback targets"
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
"dist/targets/*/adapter.json",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"security/permission_policy.json"
],
"privacy_contract": [
"Do not mark native_enforcement true for metadata-only fallbacks.",
"Keep residual risks visible for targets that still rely on operator enforcement."
],
"observed_state": {
"native_enforcement_count": 0,
"metadata_fallback_count": 4,
"residual_risk_count": 4,
"failure_count": 0,
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement."
},
{
"key": "native-client-telemetry",
"label": "Native Client Telemetry",
"category": "external",
"owner": "Browser/Chrome/IDE/provider client integrator",
"status": "pending",
"source_status": "external_required",
"current": "external source events 0; adoption samples 0",
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
"provenance_requirements": [
"real external client source",
"metadata-only event",
"local-first import path"
],
"success_checks": [
"reports/adoption_drift_report.json summary.source_types.external > 0",
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
],
"evidence_artifacts": [
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.md",
"reports/telemetry_hook_recipes.json",
"scripts/telemetry_native_host.py"
],
"privacy_contract": [
"Telemetry must remain metadata-only and local-first.",
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
],
"observed_state": {
"external_source_events": 0,
"adoption_sample_count": 0,
"raw_content_allowed": false,
"risk_band": "low",
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Install a real client against the native host and import production metadata-only events."
}
],
"source_plan": {
"json": "reports/world_class_evidence_plan.json",
"markdown": "reports/world_class_evidence_plan.md",
"task_count": 4
}
},
"synthesis_highlights": [
"Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.",
"Borrow a review checkpoint wherever trust matters more than raw speed.",
reports/skill_atlas.json
+1
View File
@@ -115,6 +115,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
reports/skill_os2_audit.json
+3 -3
View File
@@ -145,7 +145,7 @@
"key": "benchmark-reproducibility",
"label": "Benchmark Reproducibility",
"status": "pass",
"current": "artifacts 16; missing 0; failures 3",
"current": "artifacts 17; missing 0; failures 3",
"target": "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable",
"evidence": [
{
@@ -197,7 +197,7 @@
"key": "trust-security",
"label": "Trust Security",
"status": "pass",
"current": "secrets 0; scripts 77; help failures 0",
"current": "secrets 0; scripts 78; help failures 0",
"target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable",
"evidence": [
{
@@ -281,7 +281,7 @@
"key": "registry-distribution",
"label": "Registry Distribution",
"status": "pass",
"current": "zip entries 517; install failures 0; permission failures 0",
"current": "zip entries 521; install failures 0; permission failures 0",
"target": "Package metadata, archive checksum, package verification, and install simulation pass",
"evidence": [
{
reports/skill_os2_audit.md
+3 -3
View File
@@ -21,13 +21,13 @@ Generated at: `2026-06-13`
| Output Eval Lab | pass | 5 cases; delta 100.0; exec 10; blind 5 | with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy | Add more real-file and adversarial holdout cases as usage grows. |
| Provider Holdout | external-required | model-executed 0; token-observed 0 | At least one real provider-backed holdout run with observed model/timing/token metadata | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. |
| Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. |
| Benchmark Reproducibility | pass | artifacts 16; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
| Benchmark Reproducibility | pass | artifacts 17; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. |
| Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. |
| Trust Security | pass | secrets 0; scripts 77; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Trust Security | pass | secrets 0; scripts 78; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. |
| Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4 | Packaged adapters expose explicit permission metadata and residual risks | Preserve residual-risk notes until real native enforcement exists. |
| Native Permission Enforcement | external-required | native-enforced targets 0 | At least one target/client enforces approved permissions at runtime | Integrate a real client or installer runtime guard before claiming native permission enforcement. |
| Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. |
| Registry Distribution | pass | zip entries 517; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
| Registry Distribution | pass | zip entries 521; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. |
| Review Studio | pass | decision review; warnings 2; score 92 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. |
| Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. |
| Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. |
reports/upgrade_check.json
+2 -2
View File
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a"
"to": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c"
"to": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f"
}
]
},
reports/world_class_evidence_ledger.json
+204
View File
@@ -0,0 +1,204 @@
{
"schema_version": "1.0",
"ok": true,
"generated_at": "2026-06-13",
"skill_dir": ".",
"summary": {
"ledger_entry_count": 4,
"accepted_count": 0,
"pending_count": 4,
"human_pending_count": 1,
"external_pending_count": 3,
"overclaim_guard_active": true,
"ready_to_claim_world_class": false,
"decision": "evidence-pending"
},
"entries": [
{
"key": "provider-holdout",
"label": "Provider Holdout",
"category": "external",
"owner": "operator with provider credentials",
"status": "pending",
"source_status": "external_required",
"current": "model-executed 0; token-observed 0",
"objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.",
"provenance_requirements": [
"provider-backed model run",
"observed timing",
"observed token metadata"
],
"success_checks": [
"reports/output_execution_runs.json summary.model_executed_count > 0",
"reports/output_execution_runs.json summary.timing_observed_count > 0",
"reports/output_execution_runs.json summary.token_observed_count > 0",
"reports/skill_os2_audit.json item provider-holdout status becomes pass"
],
"evidence_artifacts": [
"reports/output_execution_runs.json",
"reports/output_execution_runs.md",
"reports/skill_os2_audit.json"
],
"privacy_contract": [
"Do not commit provider credentials or environment dumps.",
"The output execution report records output hashes and aggregate run metadata, not raw provider prompts."
],
"observed_state": {
"model_executed_count": 0,
"timing_observed_count": 10,
"token_observed_count": 0,
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence."
},
{
"key": "human-adjudication",
"label": "Human Adjudication",
"category": "human",
"owner": "human reviewer",
"status": "pending",
"source_status": "human_required",
"current": "0/5 decisions; pending 5",
"objective": "Record real blind A/B reviewer decisions before claiming human output review completion.",
"provenance_requirements": [
"real reviewer identity",
"blind A/B decisions",
"answer key unopened until decisions exist"
],
"success_checks": [
"reports/output_review_adjudication.json summary.pending_count == 0",
"reports/output_review_adjudication.json summary.judgment_count == summary.pair_count",
"reports/output_review_adjudication.json summary.invalid_decision_count == 0",
"reports/skill_os2_audit.json item human-adjudication status becomes pass"
],
"evidence_artifacts": [
"reports/output_blind_review_pack.md",
"reports/output_review_decisions.json",
"reports/output_review_adjudication.json",
"reports/output_review_adjudication.md"
],
"privacy_contract": [
"Reviewer decisions should not include raw user data or private customer detail.",
"Keep the answer key separate until after decisions are recorded."
],
"observed_state": {
"pair_count": 5,
"judgment_count": 0,
"pending_count": 5,
"invalid_decision_count": 0,
"answer_revealed_count": 0,
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Record real A/B choices in the decision template, then regenerate adjudication."
},
{
"key": "native-permission-enforcement",
"label": "Native Permission Enforcement",
"category": "external",
"owner": "target client or installer integrator",
"status": "pending",
"source_status": "external_required",
"current": "native-enforced targets 0",
"objective": "Prove at least one target or installer enforces approved high-permission capabilities at runtime.",
"provenance_requirements": [
"real target or installer guard",
"native enforcement flag",
"residual risk retained for fallback targets"
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
"dist/targets/*/adapter.json",
"reports/runtime_permission_probes.json",
"reports/runtime_permission_probes.md",
"security/permission_policy.json"
],
"privacy_contract": [
"Do not mark native_enforcement true for metadata-only fallbacks.",
"Keep residual risks visible for targets that still rely on operator enforcement."
],
"observed_state": {
"native_enforcement_count": 0,
"metadata_fallback_count": 4,
"residual_risk_count": 4,
"failure_count": 0,
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement."
},
{
"key": "native-client-telemetry",
"label": "Native Client Telemetry",
"category": "external",
"owner": "Browser/Chrome/IDE/provider client integrator",
"status": "pending",
"source_status": "external_required",
"current": "external source events 0; adoption samples 0",
"objective": "Import production metadata-only events from a real external client into the local drift loop.",
"provenance_requirements": [
"real external client source",
"metadata-only event",
"local-first import path"
],
"success_checks": [
"reports/adoption_drift_report.json summary.source_types.external > 0",
"reports/adoption_drift_report.json summary.adoption_sample_count > 0",
"reports/skill_os2_audit.json item native-client-telemetry status becomes pass"
],
"evidence_artifacts": [
"reports/adoption_drift_report.json",
"reports/adoption_drift_report.md",
"reports/telemetry_hook_recipes.json",
"scripts/telemetry_native_host.py"
],
"privacy_contract": [
"Telemetry must remain metadata-only and local-first.",
"Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field."
],
"observed_state": {
"external_source_events": 0,
"adoption_sample_count": 0,
"raw_content_allowed": false,
"risk_band": "low",
"accepted": false
},
"anti_overclaim": {
"planned_work_counts_as_evidence": false,
"metadata_fallback_counts_as_native_enforcement": false,
"pending_review_counts_as_human_decision": false,
"local_command_runner_counts_as_provider_model": false
},
"next_action": "Install a real client against the native host and import production metadata-only events."
}
],
"source_plan": {
"json": "reports/world_class_evidence_plan.json",
"markdown": "reports/world_class_evidence_plan.md",
"task_count": 4
},
"artifacts": {
"json": "reports/world_class_evidence_ledger.json",
"markdown": "reports/world_class_evidence_ledger.md"
}
}
reports/world_class_evidence_ledger.md
+119
View File
@@ -0,0 +1,119 @@
# World-Class Evidence Ledger
Generated at: `2026-06-13`
## Summary
- decision: `evidence-pending`
- ready to claim world-class: `false`
- entries: `4`
- accepted: `0`
- pending: `4`
- human pending: `1`
- external pending: `3`
- overclaim guard active: `true`
This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.
## Ledger
| Evidence | Status | Category | Current | Next action |
| --- | --- | --- | --- | --- |
| `provider-holdout` | `pending` | `external` | model-executed 0; token-observed 0 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. |
| `human-adjudication` | `pending` | `human` | 0/5 decisions; pending 5 | Record real A/B choices in the decision template, then regenerate adjudication. |
| `native-permission-enforcement` | `pending` | `external` | native-enforced targets 0 | Integrate a real client or installer runtime guard before claiming native permission enforcement. |
| `native-client-telemetry` | `pending` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. |
## Provider Holdout
- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.
- source status: `external_required`
- observed state: `{"model_executed_count": 0, "timing_observed_count": 10, "token_observed_count": 0, "accepted": false}`
### Provenance Requirements
- provider-backed model run
- observed timing
- observed token metadata
### Success Checks
- reports/output_execution_runs.json summary.model_executed_count > 0
- reports/output_execution_runs.json summary.timing_observed_count > 0
- reports/output_execution_runs.json summary.token_observed_count > 0
- reports/skill_os2_audit.json item provider-holdout status becomes pass
### Privacy Contract
- Do not commit provider credentials or environment dumps.
- The output execution report records output hashes and aggregate run metadata, not raw provider prompts.
## Human Adjudication
- objective: Record real blind A/B reviewer decisions before claiming human output review completion.
- source status: `human_required`
- observed state: `{"pair_count": 5, "judgment_count": 0, "pending_count": 5, "invalid_decision_count": 0, "answer_revealed_count": 0, "accepted": false}`
### Provenance Requirements
- real reviewer identity
- blind A/B decisions
- answer key unopened until decisions exist
### Success Checks
- reports/output_review_adjudication.json summary.pending_count == 0
- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count
- reports/output_review_adjudication.json summary.invalid_decision_count == 0
- reports/skill_os2_audit.json item human-adjudication status becomes pass
### Privacy Contract
- Reviewer decisions should not include raw user data or private customer detail.
- Keep the answer key separate until after decisions are recorded.
## Native Permission Enforcement
- objective: Prove at least one target or installer enforces approved high-permission capabilities at runtime.
- source status: `external_required`
- observed state: `{"native_enforcement_count": 0, "metadata_fallback_count": 4, "residual_risk_count": 4, "failure_count": 0, "accepted": false}`
### Provenance Requirements
- real target or installer guard
- native enforcement flag
- residual risk retained for fallback targets
### Success Checks
- reports/runtime_permission_probes.json summary.native_enforcement_count > 0
- reports/runtime_permission_probes.json summary.failure_count == 0
- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
### Privacy Contract
- Do not mark native_enforcement true for metadata-only fallbacks.
- Keep residual risks visible for targets that still rely on operator enforcement.
## Native Client Telemetry
- objective: Import production metadata-only events from a real external client into the local drift loop.
- source status: `external_required`
- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}`
### Provenance Requirements
- real external client source
- metadata-only event
- local-first import path
### Success Checks
- reports/adoption_drift_report.json summary.source_types.external > 0
- reports/adoption_drift_report.json summary.adoption_sample_count > 0
- reports/skill_os2_audit.json item native-client-telemetry status becomes pass
### Privacy Contract
- Telemetry must remain metadata-only and local-first.
- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field.
reports/world_class_evidence_plan.json
+3 -2
View File
@@ -93,7 +93,7 @@
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.fail_count == 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass"
],
"evidence_artifacts": [
@@ -148,6 +148,7 @@
},
"artifacts": {
"json": "reports/world_class_evidence_plan.json",
"markdown": "reports/world_class_evidence_plan.md"
"markdown": "reports/world_class_evidence_plan.md",
"ledger": "reports/world_class_evidence_ledger.md"
}
}
reports/world_class_evidence_plan.md
+1 -1
View File
@@ -98,7 +98,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It
### Success Checks
- reports/runtime_permission_probes.json summary.native_enforcement_count > 0
- reports/runtime_permission_probes.json summary.fail_count == 0
- reports/runtime_permission_probes.json summary.failure_count == 0
- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass
### Evidence Artifacts
scripts/ci_test.py
+1
View File
@@ -50,6 +50,7 @@ DEFAULT_TARGETS = [
"review-studio-check",
"skill-os2-audit-check",
"world-class-evidence-check",
"world-class-ledger-check",
"benchmark-reproducibility-check",
"feedback-check",
"adoption-drift-check",
scripts/render_benchmark_reproducibility.py
+8
View File
@@ -38,6 +38,7 @@ REQUIRED_ARTIFACTS = [
("install_simulation", "reports/install_simulation.json"),
("skill_os2_audit", "reports/skill_os2_audit.json"),
("world_class_evidence_plan", "reports/world_class_evidence_plan.json"),
("world_class_evidence_ledger", "reports/world_class_evidence_ledger.json"),
]
REPRODUCTION_COMMANDS = [
@@ -111,6 +112,11 @@ REPRODUCTION_COMMANDS = [
"command": "python3 scripts/yao.py world-class-evidence .",
"evidence": "reports/world_class_evidence_plan.json",
},
{
"label": "world-class evidence ledger",
"command": "python3 scripts/yao.py world-class-ledger .",
"evidence": "reports/world_class_evidence_ledger.json",
},
{
"label": "full ci",
"command": "make ci-test",
@@ -221,6 +227,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
output_review = load_json(reports / "output_review_adjudication.json")
skill_os2 = load_json(reports / "skill_os2_audit.json")
world_class_plan = load_json(reports / "world_class_evidence_plan.json")
world_class_ledger = load_json(reports / "world_class_evidence_ledger.json")
methodology = methodology_check(reports / "benchmark_methodology.md")
artifacts = [artifact_record(skill_dir, label, rel) for label, rel in REQUIRED_ARTIFACTS]
missing_artifacts = [item["path"] for item in artifacts if not item["exists"]]
@@ -265,6 +272,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
"world_class_ready": bool(skill_os2.get("summary", {}).get("world_class_ready", False)),
"world_class_open_gap_count": skill_os2.get("summary", {}).get("open_gap_count", 0),
"world_class_task_count": world_class_plan.get("summary", {}).get("task_count", 0),
"world_class_ledger_pending_count": world_class_ledger.get("summary", {}).get("pending_count", 0),
"working_tree_dirty": status.get("dirty"),
"changed_file_count": status.get("changed_file_count"),
},
scripts/render_review_studio.py
+2
View File
@@ -135,6 +135,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]:
"review_waivers": "reports/review_waivers.md",
"review_annotations": "reports/review_annotations.md",
"world_class_evidence_plan": "reports/world_class_evidence_plan.md",
"world_class_evidence_ledger": "reports/world_class_evidence_ledger.md",
"registry_audit": "reports/registry_audit.md",
"package_verification": "reports/package_verification.md",
"install_simulation": "reports/install_simulation.md",
@@ -166,6 +167,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]:
"adoption_drift": load_json(reports / "adoption_drift_report.json"),
"review_waivers": load_json(reports / "review_waivers.json"),
"review_annotations": load_json(reports / "review_annotations.json"),
"world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"),
"registry": load_json(reports / "registry_audit.json"),
"package_verification": load_json(reports / "package_verification.json"),
"install_simulation": load_json(reports / "install_simulation.json"),
scripts/render_world_class_evidence_ledger.py
+240
View File
@@ -0,0 +1,240 @@
#!/usr/bin/env python3
import argparse
import json
from datetime import date
from pathlib import Path
from typing import Any
from render_world_class_evidence_plan import build_plan
ROOT = Path(__file__).resolve().parent.parent
SCRIPT_INTERFACE = "cli"
SCRIPT_INTERFACE_REASON = "Renders a machine-checkable ledger for world-class external and human evidence gaps."
def load_json(path: Path) -> dict[str, Any]:
if not path.exists():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except json.JSONDecodeError:
return {}
return payload if isinstance(payload, dict) else {}
def rel_path(path: Path, root: Path) -> str:
try:
return str(path.resolve().relative_to(root.resolve()))
except ValueError:
return str(path.resolve())
def provider_state(skill_dir: Path) -> dict[str, Any]:
summary = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {})
return {
"model_executed_count": summary.get("model_executed_count", 0),
"timing_observed_count": summary.get("timing_observed_count", 0),
"token_observed_count": summary.get("token_observed_count", 0),
"accepted": (
summary.get("model_executed_count", 0) > 0
and summary.get("timing_observed_count", 0) > 0
and summary.get("token_observed_count", 0) > 0
),
}
def human_state(skill_dir: Path) -> dict[str, Any]:
summary = load_json(skill_dir / "reports" / "output_review_adjudication.json").get("summary", {})
return {
"pair_count": summary.get("pair_count", 0),
"judgment_count": summary.get("judgment_count", 0),
"pending_count": summary.get("pending_count", 0),
"invalid_decision_count": summary.get("invalid_decision_count", 0),
"answer_revealed_count": summary.get("answer_revealed_count", 0),
"accepted": (
summary.get("pair_count", 0) > 0
and summary.get("pending_count", 0) == 0
and summary.get("judgment_count", 0) == summary.get("pair_count", 0)
and summary.get("invalid_decision_count", 0) == 0
),
}
def permission_state(skill_dir: Path) -> dict[str, Any]:
summary = load_json(skill_dir / "reports" / "runtime_permission_probes.json").get("summary", {})
return {
"native_enforcement_count": summary.get("native_enforcement_count", 0),
"metadata_fallback_count": summary.get("metadata_fallback_count", 0),
"residual_risk_count": summary.get("residual_risk_count", 0),
"failure_count": summary.get("failure_count", 0),
"accepted": summary.get("native_enforcement_count", 0) > 0 and summary.get("failure_count", 0) == 0,
}
def telemetry_state(skill_dir: Path) -> dict[str, Any]:
payload = load_json(skill_dir / "reports" / "adoption_drift_report.json")
summary = payload.get("summary", {})
source_types = summary.get("source_types", {}) if isinstance(summary.get("source_types", {}), dict) else {}
privacy = payload.get("privacy_contract", {})
return {
"external_source_events": source_types.get("external", 0),
"adoption_sample_count": summary.get("adoption_sample_count", 0),
"raw_content_allowed": privacy.get("raw_content_allowed"),
"risk_band": summary.get("risk_band", "missing"),
"accepted": (
source_types.get("external", 0) > 0
and summary.get("adoption_sample_count", 0) > 0
and privacy.get("raw_content_allowed") is False
),
}
STATE_LOADERS = {
"provider-holdout": provider_state,
"human-adjudication": human_state,
"native-permission-enforcement": permission_state,
"native-client-telemetry": telemetry_state,
}
PROVENANCE_REQUIREMENTS = {
"provider-holdout": ["provider-backed model run", "observed timing", "observed token metadata"],
"human-adjudication": ["real reviewer identity", "blind A/B decisions", "answer key unopened until decisions exist"],
"native-permission-enforcement": ["real target or installer guard", "native enforcement flag", "residual risk retained for fallback targets"],
"native-client-telemetry": ["real external client source", "metadata-only event", "local-first import path"],
}
def build_entry(skill_dir: Path, task: dict[str, Any]) -> dict[str, Any]:
state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir)
accepted = bool(state.get("accepted"))
return {
"key": task["key"],
"label": task["label"],
"category": task["category"],
"owner": task["owner"],
"status": "accepted" if accepted else "pending",
"source_status": task["status"],
"current": task["current"],
"objective": task["objective"],
"provenance_requirements": PROVENANCE_REQUIREMENTS.get(task["key"], ["release reviewer evidence"]),
"success_checks": task["success_checks"],
"evidence_artifacts": task["evidence_artifacts"],
"privacy_contract": task["privacy_contract"],
"observed_state": state,
"anti_overclaim": {
"planned_work_counts_as_evidence": False,
"metadata_fallback_counts_as_native_enforcement": False,
"pending_review_counts_as_human_decision": False,
"local_command_runner_counts_as_provider_model": False,
},
"next_action": task["audit_next_action"],
}
def build_ledger(skill_dir: Path, generated_at: str) -> dict[str, Any]:
plan = build_plan(skill_dir, generated_at)
entries = [build_entry(skill_dir, task) for task in plan["tasks"]]
accepted_count = sum(1 for entry in entries if entry["status"] == "accepted")
pending_count = len(entries) - accepted_count
human_pending_count = sum(1 for entry in entries if entry["category"] == "human" and entry["status"] == "pending")
external_pending_count = sum(1 for entry in entries if entry["category"] == "external" and entry["status"] == "pending")
ready = pending_count == 0 and plan["summary"].get("world_class_ready") is True
return {
"schema_version": "1.0",
"ok": True,
"generated_at": generated_at,
"skill_dir": rel_path(skill_dir, ROOT),
"summary": {
"ledger_entry_count": len(entries),
"accepted_count": accepted_count,
"pending_count": pending_count,
"human_pending_count": human_pending_count,
"external_pending_count": external_pending_count,
"overclaim_guard_active": True,
"ready_to_claim_world_class": ready,
"decision": "ready-for-completion-audit" if ready else "evidence-pending",
},
"entries": entries,
"source_plan": {
"json": "reports/world_class_evidence_plan.json",
"markdown": "reports/world_class_evidence_plan.md",
"task_count": plan["summary"].get("task_count", 0),
},
"artifacts": {
"json": "reports/world_class_evidence_ledger.json",
"markdown": "reports/world_class_evidence_ledger.md",
},
}
def render_markdown(ledger: dict[str, Any]) -> str:
summary = ledger["summary"]
lines = [
"# World-Class Evidence Ledger",
"",
f"Generated at: `{ledger['generated_at']}`",
"",
"## Summary",
"",
f"- decision: `{summary['decision']}`",
f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`",
f"- entries: `{summary['ledger_entry_count']}`",
f"- accepted: `{summary['accepted_count']}`",
f"- pending: `{summary['pending_count']}`",
f"- human pending: `{summary['human_pending_count']}`",
f"- external pending: `{summary['external_pending_count']}`",
f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`",
"",
"This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.",
"",
"## Ledger",
"",
"| Evidence | Status | Category | Current | Next action |",
"| --- | --- | --- | --- | --- |",
]
for entry in ledger["entries"]:
current = str(entry["current"]).replace("|", "\\|")
action = str(entry["next_action"]).replace("|", "\\|")
lines.append(f"| `{entry['key']}` | `{entry['status']}` | `{entry['category']}` | {current} | {action} |")
if not ledger["entries"]:
lines.append("| `none` | `accepted` | `none` | all evidence collected | none |")
for entry in ledger["entries"]:
lines.extend(["", f"## {entry['label']}", ""])
lines.append(f"- objective: {entry['objective']}")
lines.append(f"- source status: `{entry['source_status']}`")
lines.append(f"- observed state: `{json.dumps(entry['observed_state'], ensure_ascii=False)}`")
lines.extend(["", "### Provenance Requirements", ""])
lines.extend(f"- {item}" for item in entry["provenance_requirements"])
lines.extend(["", "### Success Checks", ""])
lines.extend(f"- {item}" for item in entry["success_checks"])
lines.extend(["", "### Privacy Contract", ""])
lines.extend(f"- {item}" for item in entry["privacy_contract"])
return "\n".join(lines).rstrip() + "\n"
def main() -> None:
parser = argparse.ArgumentParser(description="Render a world-class evidence ledger.")
parser.add_argument("skill_dir", nargs="?", default=".")
parser.add_argument("--output-json", default="reports/world_class_evidence_ledger.json")
parser.add_argument("--output-md", default="reports/world_class_evidence_ledger.md")
parser.add_argument("--generated-at", default=date.today().isoformat())
args = parser.parse_args()
skill_dir = Path(args.skill_dir).resolve()
ledger = build_ledger(skill_dir, args.generated_at)
output_json = Path(args.output_json)
output_md = Path(args.output_md)
if not output_json.is_absolute():
output_json = skill_dir / output_json
if not output_md.is_absolute():
output_md = skill_dir / output_md
output_json.parent.mkdir(parents=True, exist_ok=True)
output_md.parent.mkdir(parents=True, exist_ok=True)
output_json.write_text(json.dumps(ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8")
output_md.write_text(render_markdown(ledger), encoding="utf-8")
print(json.dumps(ledger, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
scripts/render_world_class_evidence_plan.py
+2 -1
View File
@@ -77,7 +77,7 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = {
],
"success_checks": [
"reports/runtime_permission_probes.json summary.native_enforcement_count > 0",
"reports/runtime_permission_probes.json summary.fail_count == 0",
"reports/runtime_permission_probes.json summary.failure_count == 0",
"reports/skill_os2_audit.json item native-permission-enforcement status becomes pass",
],
"evidence_artifacts": [
@@ -188,6 +188,7 @@ def build_plan(skill_dir: Path, generated_at: str) -> dict[str, Any]:
"artifacts": {
"json": "reports/world_class_evidence_plan.json",
"markdown": "reports/world_class_evidence_plan.md",
"ledger": "reports/world_class_evidence_ledger.md",
},
}
scripts/skill_report_model.py
+14
View File
@@ -489,6 +489,7 @@ def build_report_model(skill_dir: Path) -> dict:
review_waivers = load_json(skill_dir / "reports" / "review_waivers.json")
review_annotations = load_json(skill_dir / "reports" / "review_annotations.json")
world_class_evidence = load_json(skill_dir / "reports" / "world_class_evidence_plan.json")
world_class_evidence_ledger = load_json(skill_dir / "reports" / "world_class_evidence_ledger.json")
compiled_targets = load_json(skill_dir / "reports" / "compiled_targets.json")
skill_ir = load_json(skill_dir / "reports" / "skill-ir.json")
if not skill_ir:
@@ -558,6 +559,13 @@ def build_report_model(skill_dir: Path) -> dict:
if (skill_dir / "reports" / "world_class_evidence_plan.md").exists():
insert_after = deliverables.index("reports/review_waivers.md") + 1
deliverables.insert(insert_after, "reports/world_class_evidence_plan.md")
if (skill_dir / "reports" / "world_class_evidence_ledger.md").exists():
insert_after = (
deliverables.index("reports/world_class_evidence_plan.md") + 1
if "reports/world_class_evidence_plan.md" in deliverables
else deliverables.index("reports/review_waivers.md") + 1
)
deliverables.insert(insert_after, "reports/world_class_evidence_ledger.md")
skill_summary = {
"name": name,
@@ -759,6 +767,12 @@ def build_report_model(skill_dir: Path) -> dict:
"tasks": world_class_evidence.get("tasks", [])[:8],
"source_audit": world_class_evidence.get("source_audit", {}),
},
"world_class_evidence_ledger": {
"ok": world_class_evidence_ledger.get("ok", False),
"summary": world_class_evidence_ledger.get("summary", {}),
"entries": world_class_evidence_ledger.get("entries", [])[:8],
"source_plan": world_class_evidence_ledger.get("source_plan", {}),
},
"synthesis_highlights": synthesis,
"artifact_design": q_review["artifact_design"],
"prompt_quality": q_review["prompt_quality"],
scripts/yao.py
+21
View File
@@ -465,6 +465,8 @@ def command_report(args: argparse.Namespace) -> int:
run_script("render_telemetry_hook_recipes.py", [str(ROOT)]),
run_script("render_review_waivers.py", [str(ROOT)]),
run_script("render_review_annotations.py", [str(ROOT)]),
run_script("render_world_class_evidence_plan.py", [str(ROOT)]),
run_script("render_world_class_evidence_ledger.py", [str(ROOT)]),
run_script("render_benchmark_reproducibility.py", [str(ROOT)]),
]
)
@@ -492,6 +494,9 @@ def command_report(args: argparse.Namespace) -> int:
"telemetry_hooks": "reports/telemetry_hook_recipes.json",
"review_waivers": "reports/review_waivers.json",
"review_annotations": "reports/review_annotations.json",
"world_class_evidence_plan": "reports/world_class_evidence_plan.json",
"world_class_evidence_ledger": "reports/world_class_evidence_ledger.json",
"benchmark_reproducibility": "reports/benchmark_reproducibility.json",
},
}
print(json.dumps(report, ensure_ascii=False, indent=2))
@@ -562,6 +567,20 @@ def command_world_class_evidence(args: argparse.Namespace) -> int:
return 0 if result["ok"] else 2
def command_world_class_ledger(args: argparse.Namespace) -> int:
skill_dir = str(Path(args.skill_dir).resolve())
cmd = [skill_dir]
if args.output_json:
cmd.extend(["--output-json", args.output_json])
if args.output_md:
cmd.extend(["--output-md", args.output_md])
if args.generated_at:
cmd.extend(["--generated-at", args.generated_at])
result = run_script("render_world_class_evidence_ledger.py", cmd)
print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2))
return 0 if result["ok"] else 2
def command_benchmark_reproducibility(args: argparse.Namespace) -> int:
skill_dir = str(Path(args.skill_dir).resolve())
cmd = [skill_dir]
@@ -1196,6 +1215,8 @@ def command_workspace_flow(args: argparse.Namespace) -> int:
{"phase": "report-refresh", "result": run_script("render_telemetry_hook_recipes.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_review_waivers.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_review_annotations.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_plan.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_world_class_evidence_ledger.py", [str(ROOT)])},
{"phase": "report-refresh", "result": run_script("render_benchmark_reproducibility.py", [str(ROOT)])},
]
)
scripts/yao_cli_parser.py
+10
View File
@@ -152,6 +152,16 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int]
world_class_evidence_cmd.add_argument("--generated-at")
world_class_evidence_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_evidence"))
world_class_ledger_cmd = subparsers.add_parser(
"world-class-ledger",
help="Render the machine-checkable ledger for world-class evidence gaps.",
)
world_class_ledger_cmd.add_argument("skill_dir", nargs="?", default=".")
world_class_ledger_cmd.add_argument("--output-json")
world_class_ledger_cmd.add_argument("--output-md")
world_class_ledger_cmd.add_argument("--generated-at")
world_class_ledger_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_ledger"))
benchmark_reproducibility_cmd = subparsers.add_parser(
"benchmark-reproducibility",
help="Render benchmark methodology, artifact, failure-disclosure, and reproduction-command evidence.",
skill-ir/examples/yao-meta-skill.json
+1
View File
@@ -159,6 +159,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
skill_atlas/catalog.json
+1
View File
@@ -71,6 +71,7 @@
"scripts/render_social_preview.py",
"scripts/render_system_model.py",
"scripts/render_telemetry_hook_recipes.py",
"scripts/render_world_class_evidence_ledger.py",
"scripts/render_world_class_evidence_plan.py",
"scripts/resource_boundary_check.py",
"scripts/review_studio_formatting.py",
tests/verify_benchmark_reproducibility.py
+15
View File
@@ -14,6 +14,19 @@ TMP = ROOT / "tests" / "tmp_benchmark_reproducibility"
def main() -> None:
shutil.rmtree(TMP, ignore_errors=True)
TMP.mkdir(parents=True, exist_ok=True)
subprocess.run(
[
sys.executable,
str(ROOT / "scripts" / "render_world_class_evidence_ledger.py"),
str(ROOT),
"--generated-at",
"2026-06-13",
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
output_json = TMP / "benchmark_reproducibility.json"
output_md = TMP / "benchmark_reproducibility.md"
proc = subprocess.run(
@@ -54,7 +67,9 @@ def main() -> None:
assert artifacts["reports/benchmark_methodology.md"]["exists"], artifacts
assert artifacts["evals/failure-cases.md"]["exists"], artifacts
assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts
assert artifacts["reports/world_class_evidence_ledger.json"]["exists"], artifacts
assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload
assert any(command["command"] == "python3 scripts/yao.py world-class-ledger ." for command in payload["reproduction_commands"]), payload
assert any("provider-backed" in item for item in payload["limitations"]), payload["limitations"]
markdown = output_md.read_text(encoding="utf-8")
assert "Benchmark Reproducibility" in markdown, markdown
tests/verify_review_studio.py
+2
View File
@@ -308,6 +308,8 @@ def main() -> None:
assert full_payload["evidence_paths"]["review_annotations"] == "reports/review_annotations.md", full_payload["evidence_paths"]
if (ROOT / "reports" / "world_class_evidence_plan.md").exists():
assert full_payload["evidence_paths"]["world_class_evidence_plan"] == "reports/world_class_evidence_plan.md", full_payload["evidence_paths"]
if (ROOT / "reports" / "world_class_evidence_ledger.md").exists():
assert full_payload["evidence_paths"]["world_class_evidence_ledger"] == "reports/world_class_evidence_ledger.md", full_payload["evidence_paths"]
assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"]
assert full_payload["data"]["output_execution"]["summary"]["command_executed_count"] == 10, full_payload["data"]["output_execution"]
assert full_payload["data"]["output_execution"]["summary"]["recorded_fixture_count"] == 0, full_payload["data"]["output_execution"]
tests/verify_skill_overview.py
+3
View File
@@ -117,6 +117,7 @@ def main() -> None:
assert "reports/output_quality_scorecard.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
assert "reports/output_blind_review_pack.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
assert "reports/benchmark_reproducibility.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
assert "reports/world_class_evidence_ledger.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
assert "reports/runtime_permission_probes.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"]
@@ -140,6 +141,8 @@ def main() -> None:
assert overview_json["output_review_adjudication"]["summary"] == {}, overview_json["output_review_adjudication"]
assert "benchmark_reproducibility" in overview_json, overview_json.keys()
assert overview_json["benchmark_reproducibility"]["summary"] == {}, overview_json["benchmark_reproducibility"]
assert "world_class_evidence_ledger" in overview_json, overview_json.keys()
assert overview_json["world_class_evidence_ledger"]["summary"] == {}, overview_json["world_class_evidence_ledger"]
assert "runtime_conformance" in overview_json, overview_json.keys()
assert "runtime_permissions" in overview_json, overview_json.keys()
assert overview_json["runtime_permissions"]["summary"] == {}, overview_json["runtime_permissions"]
tests/verify_world_class_evidence_ledger.py
+73
View File
@@ -0,0 +1,73 @@
#!/usr/bin/env python3
import json
import shutil
import subprocess
import sys
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
SCRIPT = ROOT / "scripts" / "render_world_class_evidence_ledger.py"
TMP = ROOT / "tests" / "tmp_world_class_evidence_ledger"
def main() -> None:
shutil.rmtree(TMP, ignore_errors=True)
TMP.mkdir(parents=True, exist_ok=True)
output_json = TMP / "world_class_evidence_ledger.json"
output_md = TMP / "world_class_evidence_ledger.md"
proc = subprocess.run(
[
sys.executable,
str(SCRIPT),
str(ROOT),
"--output-json",
str(output_json),
"--output-md",
str(output_md),
"--generated-at",
"2026-06-13",
],
cwd=ROOT,
capture_output=True,
text=True,
check=True,
)
payload = json.loads(proc.stdout)
assert payload["schema_version"] == "1.0", payload
assert payload["ok"] is True, payload
summary = payload["summary"]
assert summary["ledger_entry_count"] == 4, summary
assert summary["accepted_count"] == 0, summary
assert summary["pending_count"] == 4, summary
assert summary["human_pending_count"] == 1, summary
assert summary["external_pending_count"] == 3, summary
assert summary["overclaim_guard_active"] is True, summary
assert summary["ready_to_claim_world_class"] is False, summary
entries = {entry["key"]: entry for entry in payload["entries"]}
assert set(entries) == {
"provider-holdout",
"human-adjudication",
"native-permission-enforcement",
"native-client-telemetry",
}, entries
assert entries["provider-holdout"]["observed_state"]["model_executed_count"] == 0, entries["provider-holdout"]
assert entries["human-adjudication"]["observed_state"]["pending_count"] == 5, entries["human-adjudication"]
assert entries["native-permission-enforcement"]["observed_state"]["native_enforcement_count"] == 0, entries["native-permission-enforcement"]
assert any("summary.failure_count == 0" in check for check in entries["native-permission-enforcement"]["success_checks"]), entries["native-permission-enforcement"]
assert entries["native-client-telemetry"]["observed_state"]["external_source_events"] == 0, entries["native-client-telemetry"]
for entry in entries.values():
assert entry["status"] == "pending", entry
assert entry["success_checks"], entry
assert entry["privacy_contract"], entry
assert entry["anti_overclaim"]["planned_work_counts_as_evidence"] is False, entry
assert entry["anti_overclaim"]["pending_review_counts_as_human_decision"] is False, entry
markdown = output_md.read_text(encoding="utf-8")
assert "World-Class Evidence Ledger" in markdown, markdown
assert "overclaim guard active: `true`" in markdown, markdown
assert "`provider-holdout`" in markdown, markdown
print(json.dumps({"ok": True}, ensure_ascii=False, indent=2))
if __name__ == "__main__":
main()
tests/verify_world_class_evidence_plan.py
+2
View File
@@ -41,6 +41,7 @@ def main() -> None:
assert payload["summary"]["task_count"] == 4, payload
assert payload["summary"]["human_task_count"] == 1, payload
assert payload["summary"]["external_task_count"] == 3, payload
assert payload["artifacts"]["ledger"] == "reports/world_class_evidence_ledger.md", payload
tasks = {item["key"]: item for item in payload["tasks"]}
assert set(tasks) == {
"provider-holdout",
@@ -51,6 +52,7 @@ def main() -> None:
assert any("--provider-runner openai" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"]
assert any("output_review_decisions.json" in command for command in tasks["human-adjudication"]["runbook"]), tasks["human-adjudication"]
assert any("runtime-permissions" in command for command in tasks["native-permission-enforcement"]["runbook"]), tasks["native-permission-enforcement"]
assert any("summary.failure_count == 0" in check for check in tasks["native-permission-enforcement"]["success_checks"]), tasks["native-permission-enforcement"]
assert any("telemetry_native_host.py" in command for command in tasks["native-client-telemetry"]["runbook"]), tasks["native-client-telemetry"]
for task in tasks.values():
assert task["success_checks"], task
tests/verify_yao_cli.py
+18
View File
@@ -75,6 +75,7 @@ def main() -> None:
assert "review-studio" in parser_help, parser_help
assert "skill-os2-audit" in parser_help, parser_help
assert "world-class-evidence" in parser_help, parser_help
assert "world-class-ledger" in parser_help, parser_help
assert "benchmark-reproducibility" in parser_help, parser_help
assert "telemetry-import" in parser_help, parser_help
assert "telemetry-emit" in parser_help, parser_help
@@ -180,6 +181,20 @@ def main() -> None:
assert world_class_evidence_result["payload"]["summary"]["decision"] == "collect-external-evidence", world_class_evidence_result
assert world_class_evidence_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_evidence_result
world_class_ledger_result = run(
"world-class-ledger",
str(ROOT),
"--output-json",
str(tmp_root / "world_class_evidence_ledger.json"),
"--output-md",
str(tmp_root / "world_class_evidence_ledger.md"),
"--generated-at",
"2026-06-13",
)
assert world_class_ledger_result["ok"], world_class_ledger_result
assert world_class_ledger_result["payload"]["summary"]["pending_count"] == 4, world_class_ledger_result
assert world_class_ledger_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_ledger_result
benchmark_reproducibility_result = run(
"benchmark-reproducibility",
str(ROOT),
@@ -598,6 +613,9 @@ def main() -> None:
assert "output_review_adjudication" in report_result["payload"]["artifacts"], report_result
assert "adoption_drift" in report_result["payload"]["artifacts"], report_result
assert "review_annotations" in report_result["payload"]["artifacts"], report_result
assert "world_class_evidence_plan" in report_result["payload"]["artifacts"], report_result
assert "world_class_evidence_ledger" in report_result["payload"]["artifacts"], report_result
assert "benchmark_reproducibility" in report_result["payload"]["artifacts"], report_result
report_output_execution = json.loads((ROOT / "reports" / "output_execution_runs.json").read_text(encoding="utf-8"))
assert report_output_execution["summary"]["command_executed_count"] == 10, report_output_execution
assert report_output_execution["summary"]["recorded_fixture_count"] == 0, report_output_execution