diff --git a/AGENTS.md b/AGENTS.md index 86e5624c..6531880f 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/review_studio_gates.py`: Review Studio gate evaluation, release decision scoring, and gate status labels. - `scripts/render_skill_os2_audit.py`: requirement-by-requirement Skill OS 2.0 completion audit. Keep local evidence, human-required gaps, and external-required gaps separate so reports do not overclaim world-class readiness. - `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete. +- `scripts/render_world_class_evidence_ledger.py`: machine-checkable acceptance ledger for the remaining world-class evidence gaps. Keep anti-overclaim guards explicit so planned work, metadata fallbacks, pending review, and local command runners never count as final evidence. - `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable. - `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation. - `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers. diff --git a/Makefile b/Makefile index cddf3b32..2ead9bce 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check world-class-ledger-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -91,6 +91,9 @@ skill-os2-audit-check: world-class-evidence-check: $(PYTHON) tests/verify_world_class_evidence_plan.py +world-class-ledger-check: + $(PYTHON) tests/verify_world_class_evidence_ledger.py + benchmark-reproducibility-check: $(PYTHON) tests/verify_benchmark_reproducibility.py @@ -208,11 +211,11 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check world-class-ledger-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_world_class_evidence tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_world_class_evidence tests/tmp_world_class_evidence_ledger tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 47ddb5cd..f9734815 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, adoption drift, reviewer waivers, reviewer annotations, release evidence, and per-warning fix actions - a Skill OS 2.0 audit that maps each world-class requirement to current evidence, human-required gaps, and external-required gaps - a world-class evidence plan that turns remaining provider, human, native-permission, and real-client telemetry gaps into executable evidence tasks +- a world-class evidence ledger that records which external and human evidence is accepted or still pending without treating planned work as proof - a benchmark reproducibility manifest that checks methodology sections, required artifacts, failure disclosure, and reproduction commands - Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports - a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks @@ -269,7 +270,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - root governance maturity score: `90/100`; governed benchmark example: `95/100` - CJK-aware trigger matching is now covered by explicit Chinese build, packaging, eval, and near-neighbor cases - context budgets: root `944/1000`, complex benchmark `790/1000`, governed benchmark `760/1000` -- quality density: root `131.7`, complex benchmark `164.6`, governed benchmark `171.1` +- quality density: root `137.7`, complex benchmark `164.6`, governed benchmark `171.1` - regression milestones are tracked in [reports/regression_history.md](reports/regression_history.md) - description drift history is tracked in [reports/description_drift_history.md](reports/description_drift_history.md) - route confusion is tracked in [reports/route_scorecard.md](reports/route_scorecard.md) @@ -281,6 +282,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - Review Studio fix actions are embedded in [reports/review-studio.json](reports/review-studio.json) - reviewer waiver evidence is published in [reports/review_waivers.md](reports/review_waivers.md) - remaining world-class evidence tasks are published in [reports/world_class_evidence_plan.md](reports/world_class_evidence_plan.md) +- current world-class evidence acceptance state is published in [reports/world_class_evidence_ledger.md](reports/world_class_evidence_ledger.md) - benchmark reproducibility evidence is published in [reports/benchmark_reproducibility.md](reports/benchmark_reproducibility.md) - target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md) - registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md) @@ -397,7 +399,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, world-class-evidence, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, world-class-evidence, world-class-ledger, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -408,6 +410,7 @@ Utility scripts that make the meta-skill operational: - `render_regression_history.py`: turns milestone snapshots into a readable regression history report - `render_skill_os2_audit.py`: renders a requirement-by-requirement Skill OS 2.0 audit that separates landed local evidence from human-required and external-required gaps - `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence +- `render_world_class_evidence_ledger.py`: renders a machine-checkable ledger for current world-class evidence acceptance, anti-overclaim guards, provenance requirements, and privacy contracts - `render_benchmark_reproducibility.py`: renders methodology, artifact, failure-disclosure, and reproduction-command evidence for public benchmark claims - `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage diff --git a/registry/index.json b/registry/index.json index c5970482..1173e361 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 946b8f34..a8a98e54 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index 905f6e57..7da7f1cd 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,19 +3,19 @@ "ok": true, "generated_at": "2026-06-13", "skill_dir": ".", - "commit": "c2507af5267e3c1eb6802449f74236c5ab649742", + "commit": "be3fe0016116ab337a972a0bfee1b05cace7ac36", "git_status": { "available": true, "dirty": true, - "changed_file_count": 43, + "changed_file_count": 47, "sample": [ " M AGENTS.md", " M Makefile", " M README.md", " M registry/index.json", " M registry/packages/yao-meta-skill.json", - " M reports/adoption_drift_report.json", - " M reports/benchmark_methodology.md", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", " M reports/compiled_targets.json", " M reports/context_budget.json", " M reports/install_simulation.json", @@ -26,11 +26,11 @@ "summary": { "reproducibility_ready": true, "methodology_complete": true, - "required_artifact_count": 16, + "required_artifact_count": 17, "missing_artifact_count": 0, "output_case_count": 5, "failure_disclosure_count": 3, - "command_count": 15, + "command_count": 16, "command_executed_count": 10, "timing_observed_count": 10, "model_executed_count": 0, @@ -40,8 +40,9 @@ "world_class_ready": false, "world_class_open_gap_count": 4, "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 43 + "changed_file_count": 47 }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -114,8 +115,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7967, - "sha256": "b3fecad2a0c31f48238a2be03e10a870d4e9b83c9af3bdfaba4f14514e44eb71" + "bytes": 7966, + "sha256": "13838e344e1c707fea704bed95d80d499fea8e673a4a0fdc519b2671a625a4d1" }, { "label": "blind_review", @@ -149,43 +150,50 @@ "label": "trust_report", "path": "reports/security_trust_report.json", "exists": true, - "bytes": 81801, - "sha256": "b3513361608feadd78ceeda13b047aa6aa933ba6f8f94687e92831843e710075" + "bytes": 82914, + "sha256": "f709657c5ddb65011675d83f9170240722508831c0ac60ab37b57d87d8707496" }, { "label": "registry_audit", "path": "reports/registry_audit.json", "exists": true, "bytes": 3183, - "sha256": "f1ec4e2f7e8c0bab0c9e2fd5c1007d9cd368f5d4a2a05c94dbc1bce12cb67e63" + "sha256": "3966d27f6a71b2f518b9b8e74d2853ffdbc778f84669d81012f671b41dcc0544" }, { "label": "package_verification", "path": "reports/package_verification.json", "exists": true, "bytes": 19325, - "sha256": "3183e66db7e31d4188dac8bab9f42e6fe52b5e98fdb8009655caa8468d18c9d9" + "sha256": "0908995b45fb0f226a8b21f61c096ef432c95c267c7b4614efbc940edc74db8a" }, { "label": "install_simulation", "path": "reports/install_simulation.json", "exists": true, "bytes": 8557, - "sha256": "0befebd76310b0f02d131814003ff2c918802f5f6c1f356c139479ed1c1b87c6" + "sha256": "db509b728ad4d9844f146f2f7493b9180bf34ef2c75e7e2d7e57868f3b612026" }, { "label": "skill_os2_audit", "path": "reports/skill_os2_audit.json", "exists": true, "bytes": 13946, - "sha256": "7e17b3a890f92d7268b26ed15035ed191d1b478b7f96029628c9168a0a09dc33" + "sha256": "a4f4e6e665e4950289df34f6ef3ae04000666d6588fe05ca333c0809ad1acbda" }, { "label": "world_class_evidence_plan", "path": "reports/world_class_evidence_plan.json", "exists": true, - "bytes": 7555, - "sha256": "f8b28a7b02b738153ce25e4a1fc47c2b3c8236cf26ff1d53e6425816a3e48a78" + "bytes": 7614, + "sha256": "c24ea93bc38e7c5830679ab4e3a497ec06734aed274d3f8cf98befd7a31cea0e" + }, + { + "label": "world_class_evidence_ledger", + "path": "reports/world_class_evidence_ledger.json", + "exists": true, + "bytes": 8269, + "sha256": "3129158ca088b4e5fc99107f35a6a4abed3080b9eb87eb992a8c29711475394d" } ], "missing_artifacts": [], @@ -260,6 +268,11 @@ "command": "python3 scripts/yao.py world-class-evidence .", "evidence": "reports/world_class_evidence_plan.json" }, + { + "label": "world-class evidence ledger", + "command": "python3 scripts/yao.py world-class-ledger .", + "evidence": "reports/world_class_evidence_ledger.json" + }, { "label": "full ci", "command": "make ci-test", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index e5e795c9..7c76c672 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,22 +1,22 @@ # Benchmark Reproducibility Generated at: `2026-06-13` -Commit: `c2507af5267e3c1eb6802449f74236c5ab649742` +Commit: `be3fe0016116ab337a972a0bfee1b05cace7ac36` Working tree dirty at generation: `true` ## Summary - reproducibility ready: `true` - methodology complete: `true` -- required artifacts: `16` +- required artifacts: `17` - missing artifacts: `0` - output cases: `5` - disclosed failure cases: `3` -- reproduction commands: `15` +- reproduction commands: `16` - provider evidence complete: `false` - human review complete: `false` - world-class ready: `false` -- changed files at generation: `43` +- changed files at generation: `47` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. @@ -40,17 +40,18 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `b3fecad2a0c3` | +| output_execution | `reports/output_execution_runs.json` | present | `13838e344e1c` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `ddd9af90d42e` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | | runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | -| trust_report | `reports/security_trust_report.json` | present | `b3513361608f` | -| registry_audit | `reports/registry_audit.json` | present | `f1ec4e2f7e8c` | -| package_verification | `reports/package_verification.json` | present | `3183e66db7e3` | -| install_simulation | `reports/install_simulation.json` | present | `0befebd76310` | -| skill_os2_audit | `reports/skill_os2_audit.json` | present | `7e17b3a890f9` | -| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `f8b28a7b02b7` | +| trust_report | `reports/security_trust_report.json` | present | `f709657c5ddb` | +| registry_audit | `reports/registry_audit.json` | present | `3966d27f6a71` | +| package_verification | `reports/package_verification.json` | present | `0908995b45fb` | +| install_simulation | `reports/install_simulation.json` | present | `db509b728ad4` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `a4f4e6e665e4` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `c24ea93bc38e` | +| world_class_evidence_ledger | `reports/world_class_evidence_ledger.json` | present | `3129158ca088` | ## Reproduction Commands @@ -82,6 +83,8 @@ This report proves local benchmark reproducibility only. It keeps external provi - evidence: `reports/skill_os2_audit.json` - `python3 scripts/yao.py world-class-evidence .` - evidence: `reports/world_class_evidence_plan.json` +- `python3 scripts/yao.py world-class-ledger .` + - evidence: `reports/world_class_evidence_ledger.json` - `make ci-test` - evidence: `CI target output` diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index aebdaf46..9f6cac05 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -196,6 +196,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -274,7 +275,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -380,7 +381,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -425,6 +426,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -473,14 +475,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -502,7 +504,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -523,7 +525,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -534,7 +536,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -615,7 +617,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -660,6 +662,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -708,14 +711,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -735,7 +738,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -756,7 +759,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -767,7 +770,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -1020,6 +1023,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -1098,7 +1102,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -1204,7 +1208,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1249,6 +1253,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -1297,14 +1302,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1326,7 +1331,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -1347,7 +1352,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -1358,7 +1363,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -1439,7 +1444,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1484,6 +1489,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -1532,14 +1538,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -1559,7 +1565,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -1580,7 +1586,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -1591,7 +1597,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -1844,6 +1850,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -1922,7 +1929,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -2028,7 +2035,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2073,6 +2080,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -2121,14 +2129,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2150,7 +2158,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -2171,7 +2179,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -2181,7 +2189,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -2256,7 +2264,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2301,6 +2309,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -2349,14 +2358,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2376,7 +2385,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -2397,7 +2406,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -2407,7 +2416,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -2652,6 +2661,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -2730,7 +2740,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -2836,7 +2846,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2881,6 +2891,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -2929,14 +2940,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -2958,7 +2969,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -2979,7 +2990,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -2990,7 +3001,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -3064,7 +3075,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3109,6 +3120,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -3157,14 +3169,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3184,7 +3196,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -3205,7 +3217,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -3216,7 +3228,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -3460,6 +3472,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -3538,7 +3551,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -3644,7 +3657,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3689,6 +3702,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -3737,14 +3751,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3766,7 +3780,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -3787,7 +3801,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -3798,7 +3812,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -3876,7 +3890,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3921,6 +3935,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -3969,14 +3984,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -3996,7 +4011,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -4017,7 +4032,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -4028,7 +4043,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { diff --git a/reports/context_budget.json b/reports/context_budget.json index fa9a196c..8d3ca11f 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 985553, + "other_text_tokens": 1002467, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 986304, - "relevant_file_count": 431, + "estimated_total_text_tokens": 1003218, + "relevant_file_count": 436, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index e40cf6bd..e1967b9e 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "[temporary-install-root]/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 517, + "archive_entry_count": 521, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index bf174585..d49819b0 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.11, + "duration_ms": 28.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.48, + "duration_ms": 28.28, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.22, + "duration_ms": 28.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.36, + "duration_ms": 28.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.99, + "duration_ms": 28.68, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.13, + "duration_ms": 28.96, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.35, + "duration_ms": 28.39, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.95, + "duration_ms": 28.41, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.04, + "duration_ms": 28.45, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.98, + "duration_ms": 28.25, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 814050a5..1a228449 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 26.11 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 27.48 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 26.22 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.36 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 25.99 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.13 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.35 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.95 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.04 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.98 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 28.6 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 28.28 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 28.23 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.19 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 28.68 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.96 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.39 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.41 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.45 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 28.25 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 030f5a5d..b7addbbc 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", - "archive_entry_count": 517, + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", + "archive_entry_count": 521, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 97731c09..8d497c51 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a` +- Archive SHA256: `23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index da9dc693..987f9841 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 6958f1a5..50696b5f 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c` -- Archive SHA256: `4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a` +- Package SHA256: `22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f` +- Archive SHA256: `23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 2d6afcfb..9b088622 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -242,12 +242,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

77 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

517 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

78 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

521 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 77 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 78 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -273,7 +273,7 @@

证据路径

- +
@@ -313,7 +313,7 @@
-

信任报告

Secret
0
脚本数
77
网络脚本
3
Help 失败
0
包体哈希
fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c
+

信任报告

Secret
0
脚本数
78
网络脚本
3
Help 失败
0
包体哈希
22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -344,12 +344,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
517
失败数
0
警告数
0
归档哈希
4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
521
失败数
0
警告数
0
归档哈希
23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328
diff --git a/reports/review-studio.json b/reports/review-studio.json index b673f60b..220c35a5 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 77 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 78 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -252,6 +252,7 @@ "review_waivers": "reports/review_waivers.md", "review_annotations": "reports/review_annotations.md", "world_class_evidence_plan": "reports/world_class_evidence_plan.md", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "registry_audit": "reports/registry_audit.md", "package_verification": "reports/package_verification.md", "install_simulation": "reports/install_simulation.md", @@ -291,6 +292,7 @@ "reports/adoption_drift_report.md", "reports/review_waivers.md", "reports/world_class_evidence_plan.md", + "reports/world_class_evidence_ledger.md", "reports/review_annotations.md", "reports/review-studio.html", "reports/skill-overview.html" @@ -696,7 +698,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 77 + "file_count": 78 }, { "path": "evals", @@ -708,10 +710,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 179 + "file_count": 181 } ], - "file_count": 320, + "file_count": 323, "folder_count": 4, "distribution": [ { @@ -736,7 +738,7 @@ }, { "label": "scripts", - "value": 77 + "value": 78 }, { "label": "evals", @@ -744,7 +746,7 @@ }, { "label": "reports", - "value": 179 + "value": 181 } ] }, @@ -865,7 +867,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 77 + "file_count": 78 }, { "path": "evals", @@ -877,7 +879,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 179 + "file_count": 181 } ], "strengths": [ @@ -1149,11 +1151,11 @@ "summary": { "reproducibility_ready": true, "methodology_complete": true, - "required_artifact_count": 16, + "required_artifact_count": 17, "missing_artifact_count": 0, "output_case_count": 5, "failure_disclosure_count": 3, - "command_count": 15, + "command_count": 16, "command_executed_count": 10, "timing_observed_count": 10, "model_executed_count": 0, @@ -1163,10 +1165,11 @@ "world_class_ready": false, "world_class_open_gap_count": 4, "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 43 + "changed_file_count": 47 }, - "commit": "c2507af5267e3c1eb6802449f74236c5ab649742", + "commit": "be3fe0016116ab337a972a0bfee1b05cace7ac36", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -1241,8 +1244,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 161, - "script_count": 77, + "scanned_files": 162, + "script_count": 78, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -1251,18 +1254,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 54, + "file_write_script_count": 55, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 67, + "help_smoke_checked_count": 68, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 161, - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "package_hash_file_count": 162, + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" }, "skill_atlas": { "skill_count": 12, @@ -1300,8 +1303,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, "compatibility": { "openai": "pass", @@ -1332,7 +1335,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1348,8 +1351,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", - "archive_entry_count": 517, + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", + "archive_entry_count": 521, "failure_count": 0, "warning_count": 0 }, @@ -1360,7 +1363,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 517, + "archive_entry_count": 521, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1427,12 +1430,12 @@ { "field": "archive_sha256", "from": "", - "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "to": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "to": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" } ] }, @@ -1636,7 +1639,7 @@ ], "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", - "reports/runtime_permission_probes.json summary.fail_count == 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" ], "evidence_artifacts": [ @@ -1690,6 +1693,203 @@ "open_gap_count": 4 } }, + "world_class_evidence_ledger": { + "ok": true, + "summary": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "external_required", + "current": "model-executed 0; token-observed 0", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "current": "native-enforced targets 0", + "objective": "Prove at least one target or installer enforces approved high-permission capabilities at runtime.", + "provenance_requirements": [ + "real target or installer guard", + "native enforcement flag", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "security/permission_policy.json" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4 + } + }, "synthesis_highlights": [ "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", "Borrow a review checkpoint wherever trust matters more than raw speed.", @@ -3312,7 +3512,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.11, + "duration_ms": 28.6, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3340,7 +3540,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.48, + "duration_ms": 28.28, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3363,7 +3563,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.22, + "duration_ms": 28.23, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3391,7 +3591,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.36, + "duration_ms": 28.19, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3414,7 +3614,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.99, + "duration_ms": 28.68, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3442,7 +3642,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.13, + "duration_ms": 28.96, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3465,7 +3665,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.35, + "duration_ms": 28.39, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3492,7 +3692,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.95, + "duration_ms": 28.41, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3515,7 +3715,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.04, + "duration_ms": 28.45, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3544,7 +3744,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.98, + "duration_ms": 28.25, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -4049,6 +4249,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -4127,7 +4328,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -4233,7 +4434,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4278,6 +4479,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -4326,14 +4528,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4355,7 +4557,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -4376,7 +4578,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -4387,7 +4589,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -4468,7 +4670,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4513,6 +4715,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -4561,14 +4764,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -4588,7 +4791,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -4609,7 +4812,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -4620,7 +4823,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -4873,6 +5076,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -4951,7 +5155,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -5057,7 +5261,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5102,6 +5306,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -5150,14 +5355,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5179,7 +5384,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -5200,7 +5405,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -5211,7 +5416,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -5292,7 +5497,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5337,6 +5542,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -5385,14 +5591,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -5412,7 +5618,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -5433,7 +5639,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -5444,7 +5650,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -5697,6 +5903,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -5775,7 +5982,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -5881,7 +6088,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5926,6 +6133,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -5974,14 +6182,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6003,7 +6211,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -6024,7 +6232,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -6034,7 +6242,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -6109,7 +6317,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6154,6 +6362,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -6202,14 +6411,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6229,7 +6438,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -6250,7 +6459,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -6260,7 +6469,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -6505,6 +6714,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -6583,7 +6793,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -6689,7 +6899,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6734,6 +6944,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -6782,14 +6993,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -6811,7 +7022,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -6832,7 +7043,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -6843,7 +7054,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -6917,7 +7128,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6962,6 +7173,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -7010,14 +7222,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7037,7 +7249,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -7058,7 +7270,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -7069,7 +7281,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -7313,6 +7525,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -7391,7 +7604,7 @@ ], "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 } @@ -7497,7 +7710,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7542,6 +7755,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -7590,14 +7804,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7619,7 +7833,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -7640,7 +7854,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -7651,7 +7865,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -7729,7 +7943,7 @@ }, "file_write": { "required": true, - "script_count": 54, + "script_count": 55, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7774,6 +7988,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -7822,14 +8037,14 @@ }, "help_smoke": { "enabled": true, - "checked_count": 67, + "checked_count": 68, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 54, + "file_write_script_count": 55, "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 @@ -7849,7 +8064,7 @@ ], "capability_counts": { "network": 3, - "file_write": 54, + "file_write": 55, "subprocess": 8, "interactive": 0 }, @@ -7870,7 +8085,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 77, + "scripts": 78, "assets": 2, "reports": 43 }, @@ -7881,7 +8096,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 77, + "script_count": 78, "help_smoke_failed_count": 0 }, "permissions": { @@ -8521,8 +8736,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 161, - "script_count": 77, + "scanned_files": 162, + "script_count": 78, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -8531,18 +8746,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 54, + "file_write_script_count": 55, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 67, + "help_smoke_checked_count": 68, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 161, - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "package_hash_file_count": 162, + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" }, "failures": [], "warnings": [], @@ -9279,6 +9494,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a machine-checkable ledger for world-class external and human evidence gaps.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_world_class_evidence_plan.py", "interface": "cli", @@ -9674,9 +9903,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 67, - "checked_count": 67, - "passed_count": 67, + "candidate_count": 68, + "checked_count": 68, + "passed_count": 68, "failed_count": 0, "skipped_count": 10, "failed_scripts": [], @@ -10191,6 +10420,16 @@ "stdout_excerpt": "usage: render_telemetry_hook_recipes.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--output-jsonl OUTPUT_JSONL]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "command": "python3 scripts/render_world_class_evidence_ledger.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_ledger.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_world_class_evidence_plan.py", "command": "python3 scripts/render_world_class_evidence_plan.py --help", @@ -10490,6 +10729,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", @@ -10568,10 +10808,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 985553, + "other_text_tokens": 1002467, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 986304, - "relevant_file_count": 431, + "estimated_total_text_tokens": 1003218, + "relevant_file_count": 436, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -11486,6 +11726,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", @@ -12552,6 +12793,210 @@ "markdown": "reports/review_annotations.md" } }, + "world_class_evidence_ledger": { + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-13", + "skill_dir": ".", + "summary": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "external_required", + "current": "model-executed 0; token-observed 0", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "current": "native-enforced targets 0", + "objective": "Prove at least one target or installer enforces approved high-permission capabilities at runtime.", + "provenance_requirements": [ + "real target or installer guard", + "native enforcement flag", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "security/permission_policy.json" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4 + }, + "artifacts": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md" + } + }, "registry": { "ok": true, "schema_version": "2.0", @@ -12575,8 +13020,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, "compatibility": { "openai": "pass", @@ -12607,7 +13052,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -12632,7 +13077,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" } ] }, @@ -12655,8 +13100,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", - "archive_entry_count": 517, + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", + "archive_entry_count": 521, "failure_count": 0, "warning_count": 0 }, @@ -13321,7 +13766,7 @@ "installed_skill_dir": "[temporary-install-root]/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 517, + "archive_entry_count": 521, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -13663,12 +14108,12 @@ { "field": "archive_sha256", "from": "", - "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "to": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "to": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 6f6bc5d5..392780a0 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 161, - "script_count": 77, + "scanned_files": 162, + "script_count": 78, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 54, + "file_write_script_count": 55, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 67, + "help_smoke_checked_count": 68, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 161, - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "package_hash_file_count": 162, + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" }, "failures": [], "warnings": [], @@ -760,6 +760,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a machine-checkable ledger for world-class external and human evidence gaps.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_world_class_evidence_plan.py", "interface": "cli", @@ -1155,9 +1169,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 67, - "checked_count": 67, - "passed_count": 67, + "candidate_count": 68, + "checked_count": 68, + "passed_count": 68, "failed_count": 0, "skipped_count": 10, "failed_scripts": [], @@ -1672,6 +1686,16 @@ "stdout_excerpt": "usage: render_telemetry_hook_recipes.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--output-jsonl OUTPUT_JSONL]\n ", "stderr_excerpt": "" }, + { + "path": "scripts/render_world_class_evidence_ledger.py", + "command": "python3 scripts/render_world_class_evidence_ledger.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_world_class_evidence_ledger.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_world_class_evidence_plan.py", "command": "python3 scripts/render_world_class_evidence_plan.py --help", @@ -1971,6 +1995,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/run_conformance_suite.py", "scripts/run_description_optimization_suite.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 44cbfcdd..ea819731 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `161` -- Scripts: `77` +- Scanned files: `162` +- Scripts: `78` - Internal script modules: `10` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `54` +- File-write scripts: `55` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `67` +- CLI help smoke checked: `68` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `161` -- Package SHA256: `fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c` +- Package hash files: `162` +- Package SHA256: `22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `67` -- Passed scripts: `67` +- Checked scripts: `68` +- Passed scripts: `68` - Failed scripts: `none` ## Script Surface @@ -113,6 +113,7 @@ | scripts/render_social_preview.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_system_model.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_telemetry_hook_recipes.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_world_class_evidence_ledger.py | cli | True | True | True | False | False | True | False | Renders a machine-checkable ledger for world-class external and human evidence gaps. | | scripts/render_world_class_evidence_plan.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/resource_boundary_check.py | cli | False | True | True | False | False | False | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/review_studio_formatting.py | internal-module | True | False | False | False | False | False | False | Imported by render_review_studio.py to format report dictionaries as audit UI panels. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 55eacd92..6e934332 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -15,6 +15,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Bilingual Skill Overview v2 that includes these evidence surfaces. - Skill OS 2.0 Audit v0 for requirement-by-requirement evidence mapping, with explicit separation between local evidence, human-required review, and external-required provider/client integrations. - World-Class Evidence Plan v0 that turns the remaining provider holdout, human adjudication, native permission enforcement, and native client telemetry gaps into executable runbooks with success checks and privacy contracts. +- World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps. - Benchmark Reproducibility v0 that turns public benchmark methodology, required artifacts, failure disclosure, and reproduction commands into machine-checkable release evidence. - Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review. - Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers. @@ -54,6 +55,7 @@ This is still not the final world-class state. Target-native behavior contracts | Benchmark Reproducibility | `scripts/render_benchmark_reproducibility.py`, `reports/benchmark_reproducibility.md`, `tests/verify_benchmark_reproducibility.py` | v0 landed | | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | +| World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | | Review Studio 2.0 | `scripts/render_review_studio.py`, `reports/review-studio.html`, `reports/review-studio.json` with per-warning `review_actions` | v0 landed | @@ -112,13 +114,13 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `67 / 67` CLI help smoke checks passing across `77` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `68 / 68` CLI help smoke checks passing across `78` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `517` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `517` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `521` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `521` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | @@ -127,10 +129,11 @@ Next move: add real client or installer permission enforcement integration. | Review Studio | decision `review`, world-class score `92`, `13` gates, `0` blockers, `2` warnings, `2` review actions, `0` open annotation blockers | | Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps | | World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts | -| Benchmark Reproducibility | local reproducibility ready with `16` required artifacts, `0` missing artifacts, `15` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations | +| World-Class Evidence Ledger | `4` evidence entries remain pending: `1` human-required and `3` external-required; anti-overclaim guards block planned work, metadata fallbacks, pending review, and local command runners from counting as completion | +| Benchmark Reproducibility | local reproducibility ready with `17` required artifacts, `0` missing artifacts, `16` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations | | IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable | | Context Budget | initial load `944/1000`, under the production budget | -| CI | `make ci-test` target count is `63` after the Benchmark Reproducibility gate update | +| CI | `make ci-test` target count is `64` after the World-Class Evidence Ledger gate update | ## Next Highest-Leverage Moves diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 65ee8dcc..1769d644 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -621,7 +621,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
+
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/world_class_evidence_ledger.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
交付流程输入材料Skill 包体可复用能力
交付流程把用户输入、生成的包体和可复用能力放在一条线上。
@@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布320项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布323项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index 57073bd5..85cec7e1 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -30,6 +30,7 @@ "reports/adoption_drift_report.md", "reports/review_waivers.md", "reports/world_class_evidence_plan.md", + "reports/world_class_evidence_ledger.md", "reports/review_annotations.md", "reports/review-studio.html", "reports/skill-overview.html" @@ -435,7 +436,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 77 + "file_count": 78 }, { "path": "evals", @@ -447,10 +448,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 179 + "file_count": 181 } ], - "file_count": 320, + "file_count": 323, "folder_count": 4, "distribution": [ { @@ -475,7 +476,7 @@ }, { "label": "scripts", - "value": 77 + "value": 78 }, { "label": "evals", @@ -483,7 +484,7 @@ }, { "label": "reports", - "value": 179 + "value": 181 } ] }, @@ -604,7 +605,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 77 + "file_count": 78 }, { "path": "evals", @@ -616,7 +617,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 179 + "file_count": 181 } ], "strengths": [ @@ -888,11 +889,11 @@ "summary": { "reproducibility_ready": true, "methodology_complete": true, - "required_artifact_count": 16, + "required_artifact_count": 17, "missing_artifact_count": 0, "output_case_count": 5, "failure_disclosure_count": 3, - "command_count": 15, + "command_count": 16, "command_executed_count": 10, "timing_observed_count": 10, "model_executed_count": 0, @@ -902,10 +903,11 @@ "world_class_ready": false, "world_class_open_gap_count": 4, "world_class_task_count": 4, + "world_class_ledger_pending_count": 4, "working_tree_dirty": true, - "changed_file_count": 43 + "changed_file_count": 47 }, - "commit": "c2507af5267e3c1eb6802449f74236c5ab649742", + "commit": "be3fe0016116ab337a972a0bfee1b05cace7ac36", "missing_artifacts": [], "limitations": [ "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", @@ -980,8 +982,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 161, - "script_count": 77, + "scanned_files": 162, + "script_count": 78, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -990,18 +992,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 54, + "file_write_script_count": 55, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 67, + "help_smoke_checked_count": 68, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 161, - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "package_hash_file_count": 162, + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" }, "skill_atlas": { "skill_count": 12, @@ -1039,8 +1041,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "package_sha256": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, "compatibility": { "openai": "pass", @@ -1071,7 +1073,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1087,8 +1089,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", - "archive_entry_count": 517, + "archive_sha256": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328", + "archive_entry_count": 521, "failure_count": 0, "warning_count": 0 }, @@ -1099,7 +1101,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 517, + "archive_entry_count": 521, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1166,12 +1168,12 @@ { "field": "archive_sha256", "from": "", - "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "to": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "to": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" } ] }, @@ -1375,7 +1377,7 @@ ], "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", - "reports/runtime_permission_probes.json summary.fail_count == 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" ], "evidence_artifacts": [ @@ -1429,6 +1431,203 @@ "open_gap_count": 4 } }, + "world_class_evidence_ledger": { + "ok": true, + "summary": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "external_required", + "current": "model-executed 0; token-observed 0", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "current": "native-enforced targets 0", + "objective": "Prove at least one target or installer enforces approved high-permission capabilities at runtime.", + "provenance_requirements": [ + "real target or installer guard", + "native enforcement flag", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "security/permission_policy.json" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4 + } + }, "synthesis_highlights": [ "Borrow progressive disclosure: keep the entrypoint lean and move depth into references or scripts.", "Borrow a review checkpoint wherever trust matters more than raw speed.", diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 14931ef6..2a75f414 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -115,6 +115,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index d50b7957..6c87ccd8 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -145,7 +145,7 @@ "key": "benchmark-reproducibility", "label": "Benchmark Reproducibility", "status": "pass", - "current": "artifacts 16; missing 0; failures 3", + "current": "artifacts 17; missing 0; failures 3", "target": "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable", "evidence": [ { @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 77; help failures 0", + "current": "secrets 0; scripts 78; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -281,7 +281,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 517; install failures 0; permission failures 0", + "current": "zip entries 521; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 0c07a94c..e6b15596 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -21,13 +21,13 @@ Generated at: `2026-06-13` | Output Eval Lab | pass | 5 cases; delta 100.0; exec 10; blind 5 | with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy | Add more real-file and adversarial holdout cases as usage grows. | | Provider Holdout | external-required | model-executed 0; token-observed 0 | At least one real provider-backed holdout run with observed model/timing/token metadata | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | -| Benchmark Reproducibility | pass | artifacts 16; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | +| Benchmark Reproducibility | pass | artifacts 17; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 77; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 78; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4 | Packaged adapters expose explicit permission metadata and residual risks | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0 | At least one target/client enforces approved permissions at runtime | Integrate a real client or installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 517; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 521; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 2; score 92 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index ffb6bbd0..7da1a711 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" + "to": "23c245a236f450db2373b8ba7b95290df1a05b454d96890e89e25db7efaf4328" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" + "to": "22d5e9247b42c8c237c0524da9cae227ba0056807815e4174b810b178568546f" } ] }, diff --git a/reports/world_class_evidence_ledger.json b/reports/world_class_evidence_ledger.json new file mode 100644 index 00000000..3e793f2b --- /dev/null +++ b/reports/world_class_evidence_ledger.json @@ -0,0 +1,204 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-13", + "skill_dir": ".", + "summary": { + "ledger_entry_count": 4, + "accepted_count": 0, + "pending_count": 4, + "human_pending_count": 1, + "external_pending_count": 3, + "overclaim_guard_active": true, + "ready_to_claim_world_class": false, + "decision": "evidence-pending" + }, + "entries": [ + { + "key": "provider-holdout", + "label": "Provider Holdout", + "category": "external", + "owner": "operator with provider credentials", + "status": "pending", + "source_status": "external_required", + "current": "model-executed 0; token-observed 0", + "objective": "Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata.", + "provenance_requirements": [ + "provider-backed model run", + "observed timing", + "observed token metadata" + ], + "success_checks": [ + "reports/output_execution_runs.json summary.model_executed_count > 0", + "reports/output_execution_runs.json summary.timing_observed_count > 0", + "reports/output_execution_runs.json summary.token_observed_count > 0", + "reports/skill_os2_audit.json item provider-holdout status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_execution_runs.json", + "reports/output_execution_runs.md", + "reports/skill_os2_audit.json" + ], + "privacy_contract": [ + "Do not commit provider credentials or environment dumps.", + "The output execution report records output hashes and aggregate run metadata, not raw provider prompts." + ], + "observed_state": { + "model_executed_count": 0, + "timing_observed_count": 10, + "token_observed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Run provider-backed holdout cases with real credentials and commit only aggregate evidence." + }, + { + "key": "human-adjudication", + "label": "Human Adjudication", + "category": "human", + "owner": "human reviewer", + "status": "pending", + "source_status": "human_required", + "current": "0/5 decisions; pending 5", + "objective": "Record real blind A/B reviewer decisions before claiming human output review completion.", + "provenance_requirements": [ + "real reviewer identity", + "blind A/B decisions", + "answer key unopened until decisions exist" + ], + "success_checks": [ + "reports/output_review_adjudication.json summary.pending_count == 0", + "reports/output_review_adjudication.json summary.judgment_count == summary.pair_count", + "reports/output_review_adjudication.json summary.invalid_decision_count == 0", + "reports/skill_os2_audit.json item human-adjudication status becomes pass" + ], + "evidence_artifacts": [ + "reports/output_blind_review_pack.md", + "reports/output_review_decisions.json", + "reports/output_review_adjudication.json", + "reports/output_review_adjudication.md" + ], + "privacy_contract": [ + "Reviewer decisions should not include raw user data or private customer detail.", + "Keep the answer key separate until after decisions are recorded." + ], + "observed_state": { + "pair_count": 5, + "judgment_count": 0, + "pending_count": 5, + "invalid_decision_count": 0, + "answer_revealed_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." + }, + { + "key": "native-permission-enforcement", + "label": "Native Permission Enforcement", + "category": "external", + "owner": "target client or installer integrator", + "status": "pending", + "source_status": "external_required", + "current": "native-enforced targets 0", + "objective": "Prove at least one target or installer enforces approved high-permission capabilities at runtime.", + "provenance_requirements": [ + "real target or installer guard", + "native enforcement flag", + "residual risk retained for fallback targets" + ], + "success_checks": [ + "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", + "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" + ], + "evidence_artifacts": [ + "dist/targets/*/adapter.json", + "reports/runtime_permission_probes.json", + "reports/runtime_permission_probes.md", + "security/permission_policy.json" + ], + "privacy_contract": [ + "Do not mark native_enforcement true for metadata-only fallbacks.", + "Keep residual risks visible for targets that still rely on operator enforcement." + ], + "observed_state": { + "native_enforcement_count": 0, + "metadata_fallback_count": 4, + "residual_risk_count": 4, + "failure_count": 0, + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Integrate a real client or installer runtime guard before claiming native permission enforcement." + }, + { + "key": "native-client-telemetry", + "label": "Native Client Telemetry", + "category": "external", + "owner": "Browser/Chrome/IDE/provider client integrator", + "status": "pending", + "source_status": "external_required", + "current": "external source events 0; adoption samples 0", + "objective": "Import production metadata-only events from a real external client into the local drift loop.", + "provenance_requirements": [ + "real external client source", + "metadata-only event", + "local-first import path" + ], + "success_checks": [ + "reports/adoption_drift_report.json summary.source_types.external > 0", + "reports/adoption_drift_report.json summary.adoption_sample_count > 0", + "reports/skill_os2_audit.json item native-client-telemetry status becomes pass" + ], + "evidence_artifacts": [ + "reports/adoption_drift_report.json", + "reports/adoption_drift_report.md", + "reports/telemetry_hook_recipes.json", + "scripts/telemetry_native_host.py" + ], + "privacy_contract": [ + "Telemetry must remain metadata-only and local-first.", + "Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field." + ], + "observed_state": { + "external_source_events": 0, + "adoption_sample_count": 0, + "raw_content_allowed": false, + "risk_band": "low", + "accepted": false + }, + "anti_overclaim": { + "planned_work_counts_as_evidence": false, + "metadata_fallback_counts_as_native_enforcement": false, + "pending_review_counts_as_human_decision": false, + "local_command_runner_counts_as_provider_model": false + }, + "next_action": "Install a real client against the native host and import production metadata-only events." + } + ], + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": 4 + }, + "artifacts": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md" + } +} diff --git a/reports/world_class_evidence_ledger.md b/reports/world_class_evidence_ledger.md new file mode 100644 index 00000000..686cb128 --- /dev/null +++ b/reports/world_class_evidence_ledger.md @@ -0,0 +1,119 @@ +# World-Class Evidence Ledger + +Generated at: `2026-06-13` + +## Summary + +- decision: `evidence-pending` +- ready to claim world-class: `false` +- entries: `4` +- accepted: `0` +- pending: `4` +- human pending: `1` +- external pending: `3` +- overclaim guard active: `true` + +This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence. + +## Ledger + +| Evidence | Status | Category | Current | Next action | +| --- | --- | --- | --- | --- | +| `provider-holdout` | `pending` | `external` | model-executed 0; token-observed 0 | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | +| `human-adjudication` | `pending` | `human` | 0/5 decisions; pending 5 | Record real A/B choices in the decision template, then regenerate adjudication. | +| `native-permission-enforcement` | `pending` | `external` | native-enforced targets 0 | Integrate a real client or installer runtime guard before claiming native permission enforcement. | +| `native-client-telemetry` | `pending` | `external` | external source events 0; adoption samples 0 | Install a real client against the native host and import production metadata-only events. | + +## Provider Holdout + +- objective: Collect at least one provider-backed output-eval holdout run with model, timing, and token metadata. +- source status: `external_required` +- observed state: `{"model_executed_count": 0, "timing_observed_count": 10, "token_observed_count": 0, "accepted": false}` + +### Provenance Requirements + +- provider-backed model run +- observed timing +- observed token metadata + +### Success Checks + +- reports/output_execution_runs.json summary.model_executed_count > 0 +- reports/output_execution_runs.json summary.timing_observed_count > 0 +- reports/output_execution_runs.json summary.token_observed_count > 0 +- reports/skill_os2_audit.json item provider-holdout status becomes pass + +### Privacy Contract + +- Do not commit provider credentials or environment dumps. +- The output execution report records output hashes and aggregate run metadata, not raw provider prompts. + +## Human Adjudication + +- objective: Record real blind A/B reviewer decisions before claiming human output review completion. +- source status: `human_required` +- observed state: `{"pair_count": 5, "judgment_count": 0, "pending_count": 5, "invalid_decision_count": 0, "answer_revealed_count": 0, "accepted": false}` + +### Provenance Requirements + +- real reviewer identity +- blind A/B decisions +- answer key unopened until decisions exist + +### Success Checks + +- reports/output_review_adjudication.json summary.pending_count == 0 +- reports/output_review_adjudication.json summary.judgment_count == summary.pair_count +- reports/output_review_adjudication.json summary.invalid_decision_count == 0 +- reports/skill_os2_audit.json item human-adjudication status becomes pass + +### Privacy Contract + +- Reviewer decisions should not include raw user data or private customer detail. +- Keep the answer key separate until after decisions are recorded. + +## Native Permission Enforcement + +- objective: Prove at least one target or installer enforces approved high-permission capabilities at runtime. +- source status: `external_required` +- observed state: `{"native_enforcement_count": 0, "metadata_fallback_count": 4, "residual_risk_count": 4, "failure_count": 0, "accepted": false}` + +### Provenance Requirements + +- real target or installer guard +- native enforcement flag +- residual risk retained for fallback targets + +### Success Checks + +- reports/runtime_permission_probes.json summary.native_enforcement_count > 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 +- reports/skill_os2_audit.json item native-permission-enforcement status becomes pass + +### Privacy Contract + +- Do not mark native_enforcement true for metadata-only fallbacks. +- Keep residual risks visible for targets that still rely on operator enforcement. + +## Native Client Telemetry + +- objective: Import production metadata-only events from a real external client into the local drift loop. +- source status: `external_required` +- observed state: `{"external_source_events": 0, "adoption_sample_count": 0, "raw_content_allowed": false, "risk_band": "low", "accepted": false}` + +### Provenance Requirements + +- real external client source +- metadata-only event +- local-first import path + +### Success Checks + +- reports/adoption_drift_report.json summary.source_types.external > 0 +- reports/adoption_drift_report.json summary.adoption_sample_count > 0 +- reports/skill_os2_audit.json item native-client-telemetry status becomes pass + +### Privacy Contract + +- Telemetry must remain metadata-only and local-first. +- Do not package reports/telemetry_events.jsonl or any raw prompt, output, transcript, note, or message field. diff --git a/reports/world_class_evidence_plan.json b/reports/world_class_evidence_plan.json index b968fe78..055080ab 100644 --- a/reports/world_class_evidence_plan.json +++ b/reports/world_class_evidence_plan.json @@ -93,7 +93,7 @@ ], "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", - "reports/runtime_permission_probes.json summary.fail_count == 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass" ], "evidence_artifacts": [ @@ -148,6 +148,7 @@ }, "artifacts": { "json": "reports/world_class_evidence_plan.json", - "markdown": "reports/world_class_evidence_plan.md" + "markdown": "reports/world_class_evidence_plan.md", + "ledger": "reports/world_class_evidence_ledger.md" } } diff --git a/reports/world_class_evidence_plan.md b/reports/world_class_evidence_plan.md index 19f2ff32..6aed91ff 100644 --- a/reports/world_class_evidence_plan.md +++ b/reports/world_class_evidence_plan.md @@ -98,7 +98,7 @@ This report is an execution plan for the remaining world-class evidence gaps. It ### Success Checks - reports/runtime_permission_probes.json summary.native_enforcement_count > 0 -- reports/runtime_permission_probes.json summary.fail_count == 0 +- reports/runtime_permission_probes.json summary.failure_count == 0 - reports/skill_os2_audit.json item native-permission-enforcement status becomes pass ### Evidence Artifacts diff --git a/scripts/ci_test.py b/scripts/ci_test.py index 4dc0f50e..0ebe0f73 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -50,6 +50,7 @@ DEFAULT_TARGETS = [ "review-studio-check", "skill-os2-audit-check", "world-class-evidence-check", + "world-class-ledger-check", "benchmark-reproducibility-check", "feedback-check", "adoption-drift-check", diff --git a/scripts/render_benchmark_reproducibility.py b/scripts/render_benchmark_reproducibility.py index b0839eb4..2c9ddbb5 100644 --- a/scripts/render_benchmark_reproducibility.py +++ b/scripts/render_benchmark_reproducibility.py @@ -38,6 +38,7 @@ REQUIRED_ARTIFACTS = [ ("install_simulation", "reports/install_simulation.json"), ("skill_os2_audit", "reports/skill_os2_audit.json"), ("world_class_evidence_plan", "reports/world_class_evidence_plan.json"), + ("world_class_evidence_ledger", "reports/world_class_evidence_ledger.json"), ] REPRODUCTION_COMMANDS = [ @@ -111,6 +112,11 @@ REPRODUCTION_COMMANDS = [ "command": "python3 scripts/yao.py world-class-evidence .", "evidence": "reports/world_class_evidence_plan.json", }, + { + "label": "world-class evidence ledger", + "command": "python3 scripts/yao.py world-class-ledger .", + "evidence": "reports/world_class_evidence_ledger.json", + }, { "label": "full ci", "command": "make ci-test", @@ -221,6 +227,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: output_review = load_json(reports / "output_review_adjudication.json") skill_os2 = load_json(reports / "skill_os2_audit.json") world_class_plan = load_json(reports / "world_class_evidence_plan.json") + world_class_ledger = load_json(reports / "world_class_evidence_ledger.json") methodology = methodology_check(reports / "benchmark_methodology.md") artifacts = [artifact_record(skill_dir, label, rel) for label, rel in REQUIRED_ARTIFACTS] missing_artifacts = [item["path"] for item in artifacts if not item["exists"]] @@ -265,6 +272,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: "world_class_ready": bool(skill_os2.get("summary", {}).get("world_class_ready", False)), "world_class_open_gap_count": skill_os2.get("summary", {}).get("open_gap_count", 0), "world_class_task_count": world_class_plan.get("summary", {}).get("task_count", 0), + "world_class_ledger_pending_count": world_class_ledger.get("summary", {}).get("pending_count", 0), "working_tree_dirty": status.get("dirty"), "changed_file_count": status.get("changed_file_count"), }, diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index e1ea2505..61e81a1c 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -135,6 +135,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "review_waivers": "reports/review_waivers.md", "review_annotations": "reports/review_annotations.md", "world_class_evidence_plan": "reports/world_class_evidence_plan.md", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.md", "registry_audit": "reports/registry_audit.md", "package_verification": "reports/package_verification.md", "install_simulation": "reports/install_simulation.md", @@ -166,6 +167,7 @@ def load_review_data(skill_dir: Path) -> dict[str, dict[str, Any]]: "adoption_drift": load_json(reports / "adoption_drift_report.json"), "review_waivers": load_json(reports / "review_waivers.json"), "review_annotations": load_json(reports / "review_annotations.json"), + "world_class_evidence_ledger": load_json(reports / "world_class_evidence_ledger.json"), "registry": load_json(reports / "registry_audit.json"), "package_verification": load_json(reports / "package_verification.json"), "install_simulation": load_json(reports / "install_simulation.json"), diff --git a/scripts/render_world_class_evidence_ledger.py b/scripts/render_world_class_evidence_ledger.py new file mode 100644 index 00000000..2dce9477 --- /dev/null +++ b/scripts/render_world_class_evidence_ledger.py @@ -0,0 +1,240 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import date +from pathlib import Path +from typing import Any + +from render_world_class_evidence_plan import build_plan + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a machine-checkable ledger for world-class external and human evidence gaps." + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def provider_state(skill_dir: Path) -> dict[str, Any]: + summary = load_json(skill_dir / "reports" / "output_execution_runs.json").get("summary", {}) + return { + "model_executed_count": summary.get("model_executed_count", 0), + "timing_observed_count": summary.get("timing_observed_count", 0), + "token_observed_count": summary.get("token_observed_count", 0), + "accepted": ( + summary.get("model_executed_count", 0) > 0 + and summary.get("timing_observed_count", 0) > 0 + and summary.get("token_observed_count", 0) > 0 + ), + } + + +def human_state(skill_dir: Path) -> dict[str, Any]: + summary = load_json(skill_dir / "reports" / "output_review_adjudication.json").get("summary", {}) + return { + "pair_count": summary.get("pair_count", 0), + "judgment_count": summary.get("judgment_count", 0), + "pending_count": summary.get("pending_count", 0), + "invalid_decision_count": summary.get("invalid_decision_count", 0), + "answer_revealed_count": summary.get("answer_revealed_count", 0), + "accepted": ( + summary.get("pair_count", 0) > 0 + and summary.get("pending_count", 0) == 0 + and summary.get("judgment_count", 0) == summary.get("pair_count", 0) + and summary.get("invalid_decision_count", 0) == 0 + ), + } + + +def permission_state(skill_dir: Path) -> dict[str, Any]: + summary = load_json(skill_dir / "reports" / "runtime_permission_probes.json").get("summary", {}) + return { + "native_enforcement_count": summary.get("native_enforcement_count", 0), + "metadata_fallback_count": summary.get("metadata_fallback_count", 0), + "residual_risk_count": summary.get("residual_risk_count", 0), + "failure_count": summary.get("failure_count", 0), + "accepted": summary.get("native_enforcement_count", 0) > 0 and summary.get("failure_count", 0) == 0, + } + + +def telemetry_state(skill_dir: Path) -> dict[str, Any]: + payload = load_json(skill_dir / "reports" / "adoption_drift_report.json") + summary = payload.get("summary", {}) + source_types = summary.get("source_types", {}) if isinstance(summary.get("source_types", {}), dict) else {} + privacy = payload.get("privacy_contract", {}) + return { + "external_source_events": source_types.get("external", 0), + "adoption_sample_count": summary.get("adoption_sample_count", 0), + "raw_content_allowed": privacy.get("raw_content_allowed"), + "risk_band": summary.get("risk_band", "missing"), + "accepted": ( + source_types.get("external", 0) > 0 + and summary.get("adoption_sample_count", 0) > 0 + and privacy.get("raw_content_allowed") is False + ), + } + + +STATE_LOADERS = { + "provider-holdout": provider_state, + "human-adjudication": human_state, + "native-permission-enforcement": permission_state, + "native-client-telemetry": telemetry_state, +} + +PROVENANCE_REQUIREMENTS = { + "provider-holdout": ["provider-backed model run", "observed timing", "observed token metadata"], + "human-adjudication": ["real reviewer identity", "blind A/B decisions", "answer key unopened until decisions exist"], + "native-permission-enforcement": ["real target or installer guard", "native enforcement flag", "residual risk retained for fallback targets"], + "native-client-telemetry": ["real external client source", "metadata-only event", "local-first import path"], +} + + +def build_entry(skill_dir: Path, task: dict[str, Any]) -> dict[str, Any]: + state = STATE_LOADERS.get(task["key"], lambda _: {"accepted": task["status"] == "pass"})(skill_dir) + accepted = bool(state.get("accepted")) + return { + "key": task["key"], + "label": task["label"], + "category": task["category"], + "owner": task["owner"], + "status": "accepted" if accepted else "pending", + "source_status": task["status"], + "current": task["current"], + "objective": task["objective"], + "provenance_requirements": PROVENANCE_REQUIREMENTS.get(task["key"], ["release reviewer evidence"]), + "success_checks": task["success_checks"], + "evidence_artifacts": task["evidence_artifacts"], + "privacy_contract": task["privacy_contract"], + "observed_state": state, + "anti_overclaim": { + "planned_work_counts_as_evidence": False, + "metadata_fallback_counts_as_native_enforcement": False, + "pending_review_counts_as_human_decision": False, + "local_command_runner_counts_as_provider_model": False, + }, + "next_action": task["audit_next_action"], + } + + +def build_ledger(skill_dir: Path, generated_at: str) -> dict[str, Any]: + plan = build_plan(skill_dir, generated_at) + entries = [build_entry(skill_dir, task) for task in plan["tasks"]] + accepted_count = sum(1 for entry in entries if entry["status"] == "accepted") + pending_count = len(entries) - accepted_count + human_pending_count = sum(1 for entry in entries if entry["category"] == "human" and entry["status"] == "pending") + external_pending_count = sum(1 for entry in entries if entry["category"] == "external" and entry["status"] == "pending") + ready = pending_count == 0 and plan["summary"].get("world_class_ready") is True + return { + "schema_version": "1.0", + "ok": True, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "summary": { + "ledger_entry_count": len(entries), + "accepted_count": accepted_count, + "pending_count": pending_count, + "human_pending_count": human_pending_count, + "external_pending_count": external_pending_count, + "overclaim_guard_active": True, + "ready_to_claim_world_class": ready, + "decision": "ready-for-completion-audit" if ready else "evidence-pending", + }, + "entries": entries, + "source_plan": { + "json": "reports/world_class_evidence_plan.json", + "markdown": "reports/world_class_evidence_plan.md", + "task_count": plan["summary"].get("task_count", 0), + }, + "artifacts": { + "json": "reports/world_class_evidence_ledger.json", + "markdown": "reports/world_class_evidence_ledger.md", + }, + } + + +def render_markdown(ledger: dict[str, Any]) -> str: + summary = ledger["summary"] + lines = [ + "# World-Class Evidence Ledger", + "", + f"Generated at: `{ledger['generated_at']}`", + "", + "## Summary", + "", + f"- decision: `{summary['decision']}`", + f"- ready to claim world-class: `{str(summary['ready_to_claim_world_class']).lower()}`", + f"- entries: `{summary['ledger_entry_count']}`", + f"- accepted: `{summary['accepted_count']}`", + f"- pending: `{summary['pending_count']}`", + f"- human pending: `{summary['human_pending_count']}`", + f"- external pending: `{summary['external_pending_count']}`", + f"- overclaim guard active: `{str(summary['overclaim_guard_active']).lower()}`", + "", + "This ledger records the current evidence state. It does not treat planned work, metadata fallback, pending review, or local command-runner output as world-class completion evidence.", + "", + "## Ledger", + "", + "| Evidence | Status | Category | Current | Next action |", + "| --- | --- | --- | --- | --- |", + ] + for entry in ledger["entries"]: + current = str(entry["current"]).replace("|", "\\|") + action = str(entry["next_action"]).replace("|", "\\|") + lines.append(f"| `{entry['key']}` | `{entry['status']}` | `{entry['category']}` | {current} | {action} |") + if not ledger["entries"]: + lines.append("| `none` | `accepted` | `none` | all evidence collected | none |") + for entry in ledger["entries"]: + lines.extend(["", f"## {entry['label']}", ""]) + lines.append(f"- objective: {entry['objective']}") + lines.append(f"- source status: `{entry['source_status']}`") + lines.append(f"- observed state: `{json.dumps(entry['observed_state'], ensure_ascii=False)}`") + lines.extend(["", "### Provenance Requirements", ""]) + lines.extend(f"- {item}" for item in entry["provenance_requirements"]) + lines.extend(["", "### Success Checks", ""]) + lines.extend(f"- {item}" for item in entry["success_checks"]) + lines.extend(["", "### Privacy Contract", ""]) + lines.extend(f"- {item}" for item in entry["privacy_contract"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render a world-class evidence ledger.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/world_class_evidence_ledger.json") + parser.add_argument("--output-md", default="reports/world_class_evidence_ledger.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + ledger = build_ledger(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(ledger, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(ledger), encoding="utf-8") + print(json.dumps(ledger, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_world_class_evidence_plan.py b/scripts/render_world_class_evidence_plan.py index e2b38f4d..db505aa3 100644 --- a/scripts/render_world_class_evidence_plan.py +++ b/scripts/render_world_class_evidence_plan.py @@ -77,7 +77,7 @@ TASK_TEMPLATES: dict[str, dict[str, Any]] = { ], "success_checks": [ "reports/runtime_permission_probes.json summary.native_enforcement_count > 0", - "reports/runtime_permission_probes.json summary.fail_count == 0", + "reports/runtime_permission_probes.json summary.failure_count == 0", "reports/skill_os2_audit.json item native-permission-enforcement status becomes pass", ], "evidence_artifacts": [ @@ -188,6 +188,7 @@ def build_plan(skill_dir: Path, generated_at: str) -> dict[str, Any]: "artifacts": { "json": "reports/world_class_evidence_plan.json", "markdown": "reports/world_class_evidence_plan.md", + "ledger": "reports/world_class_evidence_ledger.md", }, } diff --git a/scripts/skill_report_model.py b/scripts/skill_report_model.py index 13606ec4..efab2023 100644 --- a/scripts/skill_report_model.py +++ b/scripts/skill_report_model.py @@ -489,6 +489,7 @@ def build_report_model(skill_dir: Path) -> dict: review_waivers = load_json(skill_dir / "reports" / "review_waivers.json") review_annotations = load_json(skill_dir / "reports" / "review_annotations.json") world_class_evidence = load_json(skill_dir / "reports" / "world_class_evidence_plan.json") + world_class_evidence_ledger = load_json(skill_dir / "reports" / "world_class_evidence_ledger.json") compiled_targets = load_json(skill_dir / "reports" / "compiled_targets.json") skill_ir = load_json(skill_dir / "reports" / "skill-ir.json") if not skill_ir: @@ -558,6 +559,13 @@ def build_report_model(skill_dir: Path) -> dict: if (skill_dir / "reports" / "world_class_evidence_plan.md").exists(): insert_after = deliverables.index("reports/review_waivers.md") + 1 deliverables.insert(insert_after, "reports/world_class_evidence_plan.md") + if (skill_dir / "reports" / "world_class_evidence_ledger.md").exists(): + insert_after = ( + deliverables.index("reports/world_class_evidence_plan.md") + 1 + if "reports/world_class_evidence_plan.md" in deliverables + else deliverables.index("reports/review_waivers.md") + 1 + ) + deliverables.insert(insert_after, "reports/world_class_evidence_ledger.md") skill_summary = { "name": name, @@ -759,6 +767,12 @@ def build_report_model(skill_dir: Path) -> dict: "tasks": world_class_evidence.get("tasks", [])[:8], "source_audit": world_class_evidence.get("source_audit", {}), }, + "world_class_evidence_ledger": { + "ok": world_class_evidence_ledger.get("ok", False), + "summary": world_class_evidence_ledger.get("summary", {}), + "entries": world_class_evidence_ledger.get("entries", [])[:8], + "source_plan": world_class_evidence_ledger.get("source_plan", {}), + }, "synthesis_highlights": synthesis, "artifact_design": q_review["artifact_design"], "prompt_quality": q_review["prompt_quality"], diff --git a/scripts/yao.py b/scripts/yao.py index d80e7951..056a33fc 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -465,6 +465,8 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_telemetry_hook_recipes.py", [str(ROOT)]), run_script("render_review_waivers.py", [str(ROOT)]), run_script("render_review_annotations.py", [str(ROOT)]), + run_script("render_world_class_evidence_plan.py", [str(ROOT)]), + run_script("render_world_class_evidence_ledger.py", [str(ROOT)]), run_script("render_benchmark_reproducibility.py", [str(ROOT)]), ] ) @@ -492,6 +494,9 @@ def command_report(args: argparse.Namespace) -> int: "telemetry_hooks": "reports/telemetry_hook_recipes.json", "review_waivers": "reports/review_waivers.json", "review_annotations": "reports/review_annotations.json", + "world_class_evidence_plan": "reports/world_class_evidence_plan.json", + "world_class_evidence_ledger": "reports/world_class_evidence_ledger.json", + "benchmark_reproducibility": "reports/benchmark_reproducibility.json", }, } print(json.dumps(report, ensure_ascii=False, indent=2)) @@ -562,6 +567,20 @@ def command_world_class_evidence(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_world_class_ledger(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("render_world_class_evidence_ledger.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_benchmark_reproducibility(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -1196,6 +1215,8 @@ def command_workspace_flow(args: argparse.Namespace) -> int: {"phase": "report-refresh", "result": run_script("render_telemetry_hook_recipes.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_review_waivers.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_review_annotations.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_evidence_plan.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_world_class_evidence_ledger.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_benchmark_reproducibility.py", [str(ROOT)])}, ] ) diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index 9a6ab903..31270fe8 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -152,6 +152,16 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] world_class_evidence_cmd.add_argument("--generated-at") world_class_evidence_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_evidence")) + world_class_ledger_cmd = subparsers.add_parser( + "world-class-ledger", + help="Render the machine-checkable ledger for world-class evidence gaps.", + ) + world_class_ledger_cmd.add_argument("skill_dir", nargs="?", default=".") + world_class_ledger_cmd.add_argument("--output-json") + world_class_ledger_cmd.add_argument("--output-md") + world_class_ledger_cmd.add_argument("--generated-at") + world_class_ledger_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_ledger")) + benchmark_reproducibility_cmd = subparsers.add_parser( "benchmark-reproducibility", help="Render benchmark methodology, artifact, failure-disclosure, and reproduction-command evidence.", diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index 7f4ad6fa..f0d48158 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -159,6 +159,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 6c1bd05d..80eed121 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -71,6 +71,7 @@ "scripts/render_social_preview.py", "scripts/render_system_model.py", "scripts/render_telemetry_hook_recipes.py", + "scripts/render_world_class_evidence_ledger.py", "scripts/render_world_class_evidence_plan.py", "scripts/resource_boundary_check.py", "scripts/review_studio_formatting.py", diff --git a/tests/verify_benchmark_reproducibility.py b/tests/verify_benchmark_reproducibility.py index b9230850..9b9131b0 100644 --- a/tests/verify_benchmark_reproducibility.py +++ b/tests/verify_benchmark_reproducibility.py @@ -14,6 +14,19 @@ TMP = ROOT / "tests" / "tmp_benchmark_reproducibility" def main() -> None: shutil.rmtree(TMP, ignore_errors=True) TMP.mkdir(parents=True, exist_ok=True) + subprocess.run( + [ + sys.executable, + str(ROOT / "scripts" / "render_world_class_evidence_ledger.py"), + str(ROOT), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) output_json = TMP / "benchmark_reproducibility.json" output_md = TMP / "benchmark_reproducibility.md" proc = subprocess.run( @@ -54,7 +67,9 @@ def main() -> None: assert artifacts["reports/benchmark_methodology.md"]["exists"], artifacts assert artifacts["evals/failure-cases.md"]["exists"], artifacts assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts + assert artifacts["reports/world_class_evidence_ledger.json"]["exists"], artifacts assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload + assert any(command["command"] == "python3 scripts/yao.py world-class-ledger ." for command in payload["reproduction_commands"]), payload assert any("provider-backed" in item for item in payload["limitations"]), payload["limitations"] markdown = output_md.read_text(encoding="utf-8") assert "Benchmark Reproducibility" in markdown, markdown diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index 28fc859f..552247a6 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -308,6 +308,8 @@ def main() -> None: assert full_payload["evidence_paths"]["review_annotations"] == "reports/review_annotations.md", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_evidence_plan.md").exists(): assert full_payload["evidence_paths"]["world_class_evidence_plan"] == "reports/world_class_evidence_plan.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "world_class_evidence_ledger.md").exists(): + assert full_payload["evidence_paths"]["world_class_evidence_ledger"] == "reports/world_class_evidence_ledger.md", full_payload["evidence_paths"] assert full_payload["data"]["output_blind_review"]["summary"]["pair_count"] == 5, full_payload["data"]["output_blind_review"] assert full_payload["data"]["output_execution"]["summary"]["command_executed_count"] == 10, full_payload["data"]["output_execution"] assert full_payload["data"]["output_execution"]["summary"]["recorded_fixture_count"] == 0, full_payload["data"]["output_execution"] diff --git a/tests/verify_skill_overview.py b/tests/verify_skill_overview.py index f4f8eb1d..6d0d0406 100644 --- a/tests/verify_skill_overview.py +++ b/tests/verify_skill_overview.py @@ -117,6 +117,7 @@ def main() -> None: assert "reports/output_quality_scorecard.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/output_blind_review_pack.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/benchmark_reproducibility.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/world_class_evidence_ledger.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/runtime_permission_probes.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] @@ -140,6 +141,8 @@ def main() -> None: assert overview_json["output_review_adjudication"]["summary"] == {}, overview_json["output_review_adjudication"] assert "benchmark_reproducibility" in overview_json, overview_json.keys() assert overview_json["benchmark_reproducibility"]["summary"] == {}, overview_json["benchmark_reproducibility"] + assert "world_class_evidence_ledger" in overview_json, overview_json.keys() + assert overview_json["world_class_evidence_ledger"]["summary"] == {}, overview_json["world_class_evidence_ledger"] assert "runtime_conformance" in overview_json, overview_json.keys() assert "runtime_permissions" in overview_json, overview_json.keys() assert overview_json["runtime_permissions"]["summary"] == {}, overview_json["runtime_permissions"] diff --git a/tests/verify_world_class_evidence_ledger.py b/tests/verify_world_class_evidence_ledger.py new file mode 100644 index 00000000..25e21445 --- /dev/null +++ b/tests/verify_world_class_evidence_ledger.py @@ -0,0 +1,73 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_world_class_evidence_ledger.py" +TMP = ROOT / "tests" / "tmp_world_class_evidence_ledger" + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "world_class_evidence_ledger.json" + output_md = TMP / "world_class_evidence_ledger.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + summary = payload["summary"] + assert summary["ledger_entry_count"] == 4, summary + assert summary["accepted_count"] == 0, summary + assert summary["pending_count"] == 4, summary + assert summary["human_pending_count"] == 1, summary + assert summary["external_pending_count"] == 3, summary + assert summary["overclaim_guard_active"] is True, summary + assert summary["ready_to_claim_world_class"] is False, summary + entries = {entry["key"]: entry for entry in payload["entries"]} + assert set(entries) == { + "provider-holdout", + "human-adjudication", + "native-permission-enforcement", + "native-client-telemetry", + }, entries + assert entries["provider-holdout"]["observed_state"]["model_executed_count"] == 0, entries["provider-holdout"] + assert entries["human-adjudication"]["observed_state"]["pending_count"] == 5, entries["human-adjudication"] + assert entries["native-permission-enforcement"]["observed_state"]["native_enforcement_count"] == 0, entries["native-permission-enforcement"] + assert any("summary.failure_count == 0" in check for check in entries["native-permission-enforcement"]["success_checks"]), entries["native-permission-enforcement"] + assert entries["native-client-telemetry"]["observed_state"]["external_source_events"] == 0, entries["native-client-telemetry"] + for entry in entries.values(): + assert entry["status"] == "pending", entry + assert entry["success_checks"], entry + assert entry["privacy_contract"], entry + assert entry["anti_overclaim"]["planned_work_counts_as_evidence"] is False, entry + assert entry["anti_overclaim"]["pending_review_counts_as_human_decision"] is False, entry + markdown = output_md.read_text(encoding="utf-8") + assert "World-Class Evidence Ledger" in markdown, markdown + assert "overclaim guard active: `true`" in markdown, markdown + assert "`provider-holdout`" in markdown, markdown + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_world_class_evidence_plan.py b/tests/verify_world_class_evidence_plan.py index 146e2280..650e1c6e 100644 --- a/tests/verify_world_class_evidence_plan.py +++ b/tests/verify_world_class_evidence_plan.py @@ -41,6 +41,7 @@ def main() -> None: assert payload["summary"]["task_count"] == 4, payload assert payload["summary"]["human_task_count"] == 1, payload assert payload["summary"]["external_task_count"] == 3, payload + assert payload["artifacts"]["ledger"] == "reports/world_class_evidence_ledger.md", payload tasks = {item["key"]: item for item in payload["tasks"]} assert set(tasks) == { "provider-holdout", @@ -51,6 +52,7 @@ def main() -> None: assert any("--provider-runner openai" in command for command in tasks["provider-holdout"]["runbook"]), tasks["provider-holdout"] assert any("output_review_decisions.json" in command for command in tasks["human-adjudication"]["runbook"]), tasks["human-adjudication"] assert any("runtime-permissions" in command for command in tasks["native-permission-enforcement"]["runbook"]), tasks["native-permission-enforcement"] + assert any("summary.failure_count == 0" in check for check in tasks["native-permission-enforcement"]["success_checks"]), tasks["native-permission-enforcement"] assert any("telemetry_native_host.py" in command for command in tasks["native-client-telemetry"]["runbook"]), tasks["native-client-telemetry"] for task in tasks.values(): assert task["success_checks"], task diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 4993342a..8346596d 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -75,6 +75,7 @@ def main() -> None: assert "review-studio" in parser_help, parser_help assert "skill-os2-audit" in parser_help, parser_help assert "world-class-evidence" in parser_help, parser_help + assert "world-class-ledger" in parser_help, parser_help assert "benchmark-reproducibility" in parser_help, parser_help assert "telemetry-import" in parser_help, parser_help assert "telemetry-emit" in parser_help, parser_help @@ -180,6 +181,20 @@ def main() -> None: assert world_class_evidence_result["payload"]["summary"]["decision"] == "collect-external-evidence", world_class_evidence_result assert world_class_evidence_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_evidence_result + world_class_ledger_result = run( + "world-class-ledger", + str(ROOT), + "--output-json", + str(tmp_root / "world_class_evidence_ledger.json"), + "--output-md", + str(tmp_root / "world_class_evidence_ledger.md"), + "--generated-at", + "2026-06-13", + ) + assert world_class_ledger_result["ok"], world_class_ledger_result + assert world_class_ledger_result["payload"]["summary"]["pending_count"] == 4, world_class_ledger_result + assert world_class_ledger_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_ledger_result + benchmark_reproducibility_result = run( "benchmark-reproducibility", str(ROOT), @@ -598,6 +613,9 @@ def main() -> None: assert "output_review_adjudication" in report_result["payload"]["artifacts"], report_result assert "adoption_drift" in report_result["payload"]["artifacts"], report_result assert "review_annotations" in report_result["payload"]["artifacts"], report_result + assert "world_class_evidence_plan" in report_result["payload"]["artifacts"], report_result + assert "world_class_evidence_ledger" in report_result["payload"]["artifacts"], report_result + assert "benchmark_reproducibility" in report_result["payload"]["artifacts"], report_result report_output_execution = json.loads((ROOT / "reports" / "output_execution_runs.json").read_text(encoding="utf-8")) assert report_output_execution["summary"]["command_executed_count"] == 10, report_output_execution assert report_output_execution["summary"]["recorded_fixture_count"] == 0, report_output_execution
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder