diff --git a/AGENTS.md b/AGENTS.md index 7ea4b10..86e5624 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -68,6 +68,7 @@ Clean test-only scratch directories after verification with `find tests -maxdept - `scripts/review_studio_gates.py`: Review Studio gate evaluation, release decision scoring, and gate status labels. - `scripts/render_skill_os2_audit.py`: requirement-by-requirement Skill OS 2.0 completion audit. Keep local evidence, human-required gaps, and external-required gaps separate so reports do not overclaim world-class readiness. - `scripts/render_world_class_evidence_plan.py`: executable evidence task plan for the remaining world-class readiness gaps. Keep provider, human, native-permission, and real-client telemetry evidence requirements concrete without marking planned work as complete. +- `scripts/render_benchmark_reproducibility.py`: release-facing benchmark reproducibility manifest. Keep methodology sections, required artifacts, failure disclosure, reproduction commands, and world-class limitations machine-checkable. - `scripts/skill_report_model.py`, `scripts/skill_report_metrics.py`, `scripts/skill_report_charts.py`: skill overview data model, scoring, and inline SVG chart generation. - `scripts/yao_cli_config.py`: CLI target maps, archetype heuristics, diagnosis copy, and side-effect-free shaping helpers. - `scripts/yao_cli_parser.py`: CLI argparse command surface, flags, choices, and command handler binding. diff --git a/Makefile b/Makefile index 96841ea..cddf3b3 100644 --- a/Makefile +++ b/Makefile @@ -2,7 +2,7 @@ PYTHON ?= python3 LOCAL_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills.disabled/yao-meta-skill ACTIVE_SKILL_INSTALL_DIR ?= $(HOME)/.agents/skills/yao-meta-skill -.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean +.PHONY: eval eval-suite route-scorecard route-confusion-check description-optimization judge-blind-eval description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger results-panel regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check sync-local-install sync-active-install test ci-test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -91,6 +91,9 @@ skill-os2-audit-check: world-class-evidence-check: $(PYTHON) tests/verify_world_class_evidence_plan.py +benchmark-reproducibility-check: + $(PYTHON) tests/verify_benchmark_reproducibility.py + feedback-check: $(PYTHON) tests/verify_feedback.py @@ -205,11 +208,11 @@ sync-local-install: package-check sync-active-install: package-check $(PYTHON) scripts/sync_local_install.py --install-dir "$(ACTIVE_SKILL_INSTALL_DIR)" -test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check +test: eval eval-suite route-scorecard route-confusion-check description-optimization description-optimization-check promotion-check yao-cli-check skill-overview-check skill-report-metrics-check skill-report-charts-check skill-ir-check compiler-check output-eval-check output-execution-check output-review-adjudication-check runtime-conformance-check runtime-permission-check trust-check skill-atlas-check registry-audit-check package-verify-check install-simulation-check upgrade-check review-viewer-check review-studio-check skill-os2-audit-check world-class-evidence-check benchmark-reproducibility-check feedback-check adoption-drift-check telemetry-import-check telemetry-emit-check telemetry-hooks-check telemetry-native-host-check review-waivers-check review-annotations-check baseline-compare-check reference-scan-check github-benchmark-scan-check intent-confidence-check reference-synthesis-check output-risk-profile-check artifact-design-profile-check prompt-quality-profile-check system-model-check iteration-directions-check description-drift-history iteration-ledger regression-history context-reports portability-report portability-check failure-regression-check package-check package-failure-check security-boundary-check local-install-sync-check snapshot-check validate lint governance-check resource-boundary-check quality-check ci-test: $(PYTHON) scripts/ci_test.py clean: - rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_world_class_evidence tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md + rm -rf dist tests/tmp tests/tmp_snapshot tests/tmp_cli tests/tmp_skill_overview tests/tmp_skill_report_metrics tests/tmp_skill_report_charts tests/tmp_skill_ir tests/tmp_compile_skill tests/tmp_output_eval tests/tmp_output_execution tests/tmp_output_review_adjudication tests/tmp_conformance tests/tmp_runtime_permission tests/tmp_trust tests/tmp_skill_atlas tests/tmp_registry tests/tmp_package_verification tests/tmp_install_simulation tests/tmp_upgrade_check tests/tmp_reference_scan tests/tmp_iteration_directions tests/tmp_review_viewer tests/tmp_review_studio tests/tmp_skill_os2_audit tests/tmp_world_class_evidence tests/tmp_benchmark_reproducibility tests/tmp_feedback tests/tmp_adoption_drift tests/tmp_telemetry_import tests/tmp_telemetry_emit tests/tmp_telemetry_hooks tests/tmp_telemetry_native_host tests/tmp_review_waivers tests/tmp_review_annotations tests/tmp_github_benchmark_scan tests/tmp_intent_confidence tests/tmp_reference_synthesis tests/tmp_output_risk_profile tests/tmp_artifact_design_profile tests/tmp_prompt_quality_profile tests/tmp_system_model tests/tmp_security tests/tmp_baseline_compare.json tests/tmp_baseline_compare.md find . -type d -name __pycache__ -prune -exec rm -rf {} + diff --git a/README.md b/README.md index 1ea7d24..47ddb5c 100644 --- a/README.md +++ b/README.md @@ -25,6 +25,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl - a Review Studio 2.0 HTML gate page that combines intent, trigger, output eval, context, runtime, trust, atlas, adoption drift, reviewer waivers, reviewer annotations, release evidence, and per-warning fix actions - a Skill OS 2.0 audit that maps each world-class requirement to current evidence, human-required gaps, and external-required gaps - a world-class evidence plan that turns remaining provider, human, native-permission, and real-client telemetry gaps into executable evidence tasks +- a benchmark reproducibility manifest that checks methodology sections, required artifacts, failure disclosure, and reproduction commands - Output Eval Lab evidence with assertion grading, execution/timing/token evidence, a blind A/B review pack, a separate answer key, and reviewer adjudication reports - a runtime permission probe report that checks packaged target adapters for explicit permission metadata, native-enforcement flags, metadata fallback notes, and residual risks - a side-by-side HTML review studio for first-pass human review @@ -280,6 +281,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - Review Studio fix actions are embedded in [reports/review-studio.json](reports/review-studio.json) - reviewer waiver evidence is published in [reports/review_waivers.md](reports/review_waivers.md) - remaining world-class evidence tasks are published in [reports/world_class_evidence_plan.md](reports/world_class_evidence_plan.md) +- benchmark reproducibility evidence is published in [reports/benchmark_reproducibility.md](reports/benchmark_reproducibility.md) - target compiler evidence is published in [reports/compiled_targets.md](reports/compiled_targets.md) - registry package metadata and audit status are published in [reports/registry_audit.md](reports/registry_audit.md) - package archive verification is published in [reports/package_verification.md](reports/package_verification.md) @@ -395,7 +397,7 @@ Utility scripts that make the meta-skill operational: - `run_description_optimization_suite.py`: runs description optimization across the root skill and governed examples, then writes reusable reports and optional drift snapshots with calibration and family summaries - `promotion_checker.py`: applies promotion policy to current description candidates, writes promotion decisions, builds candidate registries, and emits iteration bundles with review stubs - `create_iteration_snapshot.py`: freezes the current promotion decision into a versioned release snapshot with review, route, and context evidence -- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, world-class-evidence, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint +- `yao.py`: unified authoring CLI that exposes init, validate, optimize-description, promote-check, review, release-snapshot, workspace-flow, report, skill-ir, compile-skill, output-exec, output-review, skill-os2-audit, world-class-evidence, benchmark-reproducibility, telemetry-emit, telemetry-hooks, telemetry-import, package, registry-audit, package-verify, install-simulate, upgrade-check, review-waivers, and test as one entrypoint - `render_description_drift_history.py`: turns description-optimization snapshots into a readable drift-history report - `build_confusion_matrix.py`: scores route confusion across tracked sibling skills and `no_route` cases, then writes a route scorecard and optional milestone snapshot - `render_iteration_ledger.py`: compresses regression milestones, description optimization drift, and route scorecards into one iteration-facing ledger @@ -406,6 +408,7 @@ Utility scripts that make the meta-skill operational: - `render_regression_history.py`: turns milestone snapshots into a readable regression history report - `render_skill_os2_audit.py`: renders a requirement-by-requirement Skill OS 2.0 audit that separates landed local evidence from human-required and external-required gaps - `render_world_class_evidence_plan.py`: renders executable evidence tasks for remaining world-class gaps without treating planned external work as completed evidence +- `render_benchmark_reproducibility.py`: renders methodology, artifact, failure-disclosure, and reproduction-command evidence for public benchmark claims - `cross_packager.py`: builds client-specific export artifacts from Skill IR plus neutral metadata, with explicit platform contracts and validation - `render_portability_report.py`: scores cross-environment portability from neutral metadata, degradation rules, and consumer validation coverage - `render_skill_overview.py`: generates the white-background bilingual HTML skill audit report with sticky four-character Chinese navigation, top-right language switch, v2 scorecard, inline SVG charts, contract boundary, quality review, risk governance, assets, and iteration roadmap diff --git a/registry/index.json b/registry/index.json index da0a277..c597048 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index 83ae467..946b8f3 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726", - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 0b7232a..9543278 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T15:09:29Z", + "generated_at": "2026-06-13T00:00:00Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/benchmark_methodology.md b/reports/benchmark_methodology.md index 91fa5ff..7922a2d 100644 --- a/reports/benchmark_methodology.md +++ b/reports/benchmark_methodology.md @@ -60,6 +60,7 @@ Recommended release evidence: git rev-parse HEAD python3 scripts/run_output_eval.py python3 scripts/export_skill_ir.py . --output-json skill-ir/examples/yao-meta-skill.json +python3 scripts/yao.py benchmark-reproducibility . make ci-test ``` diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json new file mode 100644 index 0000000..905f6e5 --- /dev/null +++ b/reports/benchmark_reproducibility.json @@ -0,0 +1,283 @@ +{ + "schema_version": "1.0", + "ok": true, + "generated_at": "2026-06-13", + "skill_dir": ".", + "commit": "c2507af5267e3c1eb6802449f74236c5ab649742", + "git_status": { + "available": true, + "dirty": true, + "changed_file_count": 43, + "sample": [ + " M AGENTS.md", + " M Makefile", + " M README.md", + " M registry/index.json", + " M registry/packages/yao-meta-skill.json", + " M reports/adoption_drift_report.json", + " M reports/benchmark_methodology.md", + " M reports/compiled_targets.json", + " M reports/context_budget.json", + " M reports/install_simulation.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md" + ] + }, + "summary": { + "reproducibility_ready": true, + "methodology_complete": true, + "required_artifact_count": 16, + "missing_artifact_count": 0, + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 15, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 0, + "token_observed_count": 0, + "human_review_complete": false, + "provider_evidence_complete": false, + "world_class_ready": false, + "world_class_open_gap_count": 4, + "world_class_task_count": 4, + "working_tree_dirty": true, + "changed_file_count": 43 + }, + "methodology": { + "path": "reports/benchmark_methodology.md", + "exists": true, + "sections": [ + { + "heading": "## Benchmark Types", + "exists": true + }, + { + "heading": "## Sample Sources", + "exists": true + }, + { + "heading": "## Evaluation Dimensions", + "exists": true + }, + { + "heading": "## Weighting Rule", + "exists": true + }, + { + "heading": "## Failure Disclosure", + "exists": true + }, + { + "heading": "## Reproduction", + "exists": true + } + ], + "missing_sections": [] + }, + "artifacts_checked": [ + { + "label": "methodology", + "path": "reports/benchmark_methodology.md", + "exists": true, + "bytes": 2715, + "sha256": "57025e0123ce5d10401c5bff376d2eeeac7943c83897ae1ad3fb22cadf790f92" + }, + { + "label": "failure_disclosure", + "path": "evals/failure-cases.md", + "exists": true, + "bytes": 889, + "sha256": "28833c0d4a217d612879d193fb5de199880dd5b3093ab5757e4315600fa4fb08" + }, + { + "label": "output_cases", + "path": "evals/output/cases.jsonl", + "exists": true, + "bytes": 6555, + "sha256": "a6ae9685711620d7203b73ace4412194ae287689f945b5139ec8ad75b9eefe04" + }, + { + "label": "output_schema", + "path": "evals/output/schema.json", + "exists": true, + "bytes": 2193, + "sha256": "8ee340c95064260c5e952be614e19841ac676162c6bf01d21b107e38cb04e0b9" + }, + { + "label": "output_scorecard", + "path": "reports/output_quality_scorecard.json", + "exists": true, + "bytes": 25530, + "sha256": "0806258a8e084b27e112537faff0de64a8519ca90cfdc78b57c0e4c08a514cca" + }, + { + "label": "output_execution", + "path": "reports/output_execution_runs.json", + "exists": true, + "bytes": 7967, + "sha256": "b3fecad2a0c31f48238a2be03e10a870d4e9b83c9af3bdfaba4f14514e44eb71" + }, + { + "label": "blind_review", + "path": "reports/output_blind_review_pack.json", + "exists": true, + "bytes": 7804, + "sha256": "bbe2db8ec2776fe289cd7d6bb78d48c2b8baa106ad37f79c15e43669b10c9390" + }, + { + "label": "review_adjudication", + "path": "reports/output_review_adjudication.json", + "exists": true, + "bytes": 2431, + "sha256": "ddd9af90d42e2858d629ebe3f1d7505a675f7d95517fcd9a965c80102a63158d" + }, + { + "label": "trigger_scorecard", + "path": "reports/route_scorecard.json", + "exists": true, + "bytes": 16961, + "sha256": "c164e83e36d0af276b6af2de2a5e026d7f0711b83eef2b5dcd0e760bd8bb28fc" + }, + { + "label": "runtime_conformance", + "path": "reports/conformance_matrix.json", + "exists": true, + "bytes": 10313, + "sha256": "8251329e663dda51472f29b7721e73d72ccbec9760d96fda022f6218a5a6e347" + }, + { + "label": "trust_report", + "path": "reports/security_trust_report.json", + "exists": true, + "bytes": 81801, + "sha256": "b3513361608feadd78ceeda13b047aa6aa933ba6f8f94687e92831843e710075" + }, + { + "label": "registry_audit", + "path": "reports/registry_audit.json", + "exists": true, + "bytes": 3183, + "sha256": "f1ec4e2f7e8c0bab0c9e2fd5c1007d9cd368f5d4a2a05c94dbc1bce12cb67e63" + }, + { + "label": "package_verification", + "path": "reports/package_verification.json", + "exists": true, + "bytes": 19325, + "sha256": "3183e66db7e31d4188dac8bab9f42e6fe52b5e98fdb8009655caa8468d18c9d9" + }, + { + "label": "install_simulation", + "path": "reports/install_simulation.json", + "exists": true, + "bytes": 8557, + "sha256": "0befebd76310b0f02d131814003ff2c918802f5f6c1f356c139479ed1c1b87c6" + }, + { + "label": "skill_os2_audit", + "path": "reports/skill_os2_audit.json", + "exists": true, + "bytes": 13946, + "sha256": "7e17b3a890f92d7268b26ed15035ed191d1b478b7f96029628c9168a0a09dc33" + }, + { + "label": "world_class_evidence_plan", + "path": "reports/world_class_evidence_plan.json", + "exists": true, + "bytes": 7555, + "sha256": "f8b28a7b02b738153ce25e4a1fc47c2b3c8236cf26ff1d53e6425816a3e48a78" + } + ], + "missing_artifacts": [], + "reproduction_commands": [ + { + "label": "source commit", + "command": "git rev-parse HEAD", + "evidence": "git commit hash" + }, + { + "label": "trigger eval", + "command": "make eval-suite", + "evidence": "reports/eval_suite.json" + }, + { + "label": "output eval", + "command": "python3 scripts/yao.py output-eval", + "evidence": "reports/output_quality_scorecard.json" + }, + { + "label": "output execution", + "command": "python3 scripts/yao.py output-exec --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'", + "evidence": "reports/output_execution_runs.json" + }, + { + "label": "blind review adjudication", + "command": "python3 scripts/yao.py output-review", + "evidence": "reports/output_review_adjudication.json" + }, + { + "label": "skill ir", + "command": "python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json", + "evidence": "skill-ir/examples/yao-meta-skill.json" + }, + { + "label": "runtime conformance", + "command": "python3 scripts/yao.py conformance .", + "evidence": "reports/conformance_matrix.json" + }, + { + "label": "trust report", + "command": "python3 scripts/yao.py trust .", + "evidence": "reports/security_trust_report.json" + }, + { + "label": "package", + "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip", + "evidence": "dist/yao-meta-skill.zip" + }, + { + "label": "package verify", + "command": "python3 scripts/yao.py package-verify . --package-dir dist --require-zip", + "evidence": "reports/package_verification.json" + }, + { + "label": "install simulate", + "command": "python3 scripts/yao.py install-simulate . --package-dir dist", + "evidence": "reports/install_simulation.json" + }, + { + "label": "registry audit", + "command": "python3 scripts/yao.py registry-audit .", + "evidence": "reports/registry_audit.json" + }, + { + "label": "skill os audit", + "command": "python3 scripts/yao.py skill-os2-audit .", + "evidence": "reports/skill_os2_audit.json" + }, + { + "label": "world-class evidence plan", + "command": "python3 scripts/yao.py world-class-evidence .", + "evidence": "reports/world_class_evidence_plan.json" + }, + { + "label": "full ci", + "command": "make ci-test", + "evidence": "CI target output" + } + ], + "failure_disclosure": { + "path": "evals/failure-cases.md", + "case_count": 3, + "policy": "Keep representative failures visible and tied to regression checks." + }, + "limitations": [ + "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close." + ], + "artifacts": { + "json": "reports/benchmark_reproducibility.json", + "markdown": "reports/benchmark_reproducibility.md" + } +} diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md new file mode 100644 index 0000000..e5e795c --- /dev/null +++ b/reports/benchmark_reproducibility.md @@ -0,0 +1,98 @@ +# Benchmark Reproducibility + +Generated at: `2026-06-13` +Commit: `c2507af5267e3c1eb6802449f74236c5ab649742` +Working tree dirty at generation: `true` + +## Summary + +- reproducibility ready: `true` +- methodology complete: `true` +- required artifacts: `16` +- missing artifacts: `0` +- output cases: `5` +- disclosed failure cases: `3` +- reproduction commands: `15` +- provider evidence complete: `false` +- human review complete: `false` +- world-class ready: `false` +- changed files at generation: `43` + +This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. + +## Methodology Sections + +| Section | Status | +| --- | --- | +| `## Benchmark Types` | present | +| `## Sample Sources` | present | +| `## Evaluation Dimensions` | present | +| `## Weighting Rule` | present | +| `## Failure Disclosure` | present | +| `## Reproduction` | present | + +## Required Artifacts + +| Label | Path | Status | SHA256 | +| --- | --- | --- | --- | +| methodology | `reports/benchmark_methodology.md` | present | `57025e0123ce` | +| failure_disclosure | `evals/failure-cases.md` | present | `28833c0d4a21` | +| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | +| output_schema | `evals/output/schema.json` | present | `8ee340c95064` | +| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | +| output_execution | `reports/output_execution_runs.json` | present | `b3fecad2a0c3` | +| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | +| review_adjudication | `reports/output_review_adjudication.json` | present | `ddd9af90d42e` | +| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | +| runtime_conformance | `reports/conformance_matrix.json` | present | `8251329e663d` | +| trust_report | `reports/security_trust_report.json` | present | `b3513361608f` | +| registry_audit | `reports/registry_audit.json` | present | `f1ec4e2f7e8c` | +| package_verification | `reports/package_verification.json` | present | `3183e66db7e3` | +| install_simulation | `reports/install_simulation.json` | present | `0befebd76310` | +| skill_os2_audit | `reports/skill_os2_audit.json` | present | `7e17b3a890f9` | +| world_class_evidence_plan | `reports/world_class_evidence_plan.json` | present | `f8b28a7b02b7` | + +## Reproduction Commands + +- `git rev-parse HEAD` + - evidence: `git commit hash` +- `make eval-suite` + - evidence: `reports/eval_suite.json` +- `python3 scripts/yao.py output-eval` + - evidence: `reports/output_quality_scorecard.json` +- `python3 scripts/yao.py output-exec --runner-command '["python3","scripts/local_output_eval_runner.py"]'` + - evidence: `reports/output_execution_runs.json` +- `python3 scripts/yao.py output-review` + - evidence: `reports/output_review_adjudication.json` +- `python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json` + - evidence: `skill-ir/examples/yao-meta-skill.json` +- `python3 scripts/yao.py conformance .` + - evidence: `reports/conformance_matrix.json` +- `python3 scripts/yao.py trust .` + - evidence: `reports/security_trust_report.json` +- `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip` + - evidence: `dist/yao-meta-skill.zip` +- `python3 scripts/yao.py package-verify . --package-dir dist --require-zip` + - evidence: `reports/package_verification.json` +- `python3 scripts/yao.py install-simulate . --package-dir dist` + - evidence: `reports/install_simulation.json` +- `python3 scripts/yao.py registry-audit .` + - evidence: `reports/registry_audit.json` +- `python3 scripts/yao.py skill-os2-audit .` + - evidence: `reports/skill_os2_audit.json` +- `python3 scripts/yao.py world-class-evidence .` + - evidence: `reports/world_class_evidence_plan.json` +- `make ci-test` + - evidence: `CI target output` + +## Failure Disclosure + +- path: `evals/failure-cases.md` +- disclosed cases: `3` +- policy: Keep representative failures visible and tied to regression checks. + +## Limits + +- Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence. +- Pending blind-review decisions are visible but do not count as human adjudication. +- World-class readiness remains false until external and human evidence gaps close. diff --git a/reports/compiled_targets.json b/reports/compiled_targets.json index dff7149..aebdaf4 100644 --- a/reports/compiled_targets.json +++ b/reports/compiled_targets.json @@ -173,6 +173,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -273,7 +274,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -379,7 +380,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -401,6 +402,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -439,9 +441,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -470,15 +473,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -499,8 +502,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -520,7 +523,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -531,7 +534,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -612,7 +615,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -634,6 +637,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -672,9 +676,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -703,15 +708,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -730,8 +735,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -751,7 +756,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -762,7 +767,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -992,6 +997,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -1092,7 +1098,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -1198,7 +1204,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1220,6 +1226,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -1258,9 +1265,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -1289,15 +1297,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -1318,8 +1326,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -1339,7 +1347,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -1350,7 +1358,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -1431,7 +1439,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -1453,6 +1461,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -1491,9 +1500,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -1522,15 +1532,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -1549,8 +1559,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -1570,7 +1580,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -1581,7 +1591,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -1811,6 +1821,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -1911,7 +1922,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -2017,7 +2028,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2039,6 +2050,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -2077,9 +2089,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -2108,15 +2121,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -2137,8 +2150,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -2158,7 +2171,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -2168,7 +2181,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -2243,7 +2256,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2265,6 +2278,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -2303,9 +2317,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -2334,15 +2349,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -2361,8 +2376,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -2382,7 +2397,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -2392,7 +2407,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -2614,6 +2629,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -2714,7 +2730,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -2820,7 +2836,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -2842,6 +2858,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -2880,9 +2897,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -2911,15 +2929,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -2940,8 +2958,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -2961,7 +2979,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -2972,7 +2990,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -3046,7 +3064,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3068,6 +3086,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -3106,9 +3125,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -3137,15 +3157,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -3164,8 +3184,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -3185,7 +3205,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -3196,7 +3216,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -3417,6 +3437,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -3517,7 +3538,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -3623,7 +3644,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3645,6 +3666,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -3683,9 +3705,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -3714,15 +3737,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -3743,8 +3766,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -3764,7 +3787,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -3775,7 +3798,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -3853,7 +3876,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -3875,6 +3898,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -3913,9 +3937,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -3944,15 +3969,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -3971,8 +3996,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -3992,7 +4017,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -4003,7 +4028,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { diff --git a/reports/context_budget.json b/reports/context_budget.json index 77842d1..fa9a196 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,10 +6,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 974109, + "other_text_tokens": 985553, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 974860, - "relevant_file_count": 425, + "estimated_total_text_tokens": 986304, + "relevant_file_count": 431, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 diff --git a/reports/install_simulation.json b/reports/install_simulation.json index 1fbbdbf..e40cf6b 100644 --- a/reports/install_simulation.json +++ b/reports/install_simulation.json @@ -8,7 +8,7 @@ "installed_skill_dir": "[temporary-install-root]/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 513, + "archive_entry_count": 517, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index e11088b..bf17458 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.18, + "duration_ms": 26.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.61, + "duration_ms": 27.48, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.45, + "duration_ms": 26.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.3, + "duration_ms": 26.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.92, + "duration_ms": 25.99, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.87, + "duration_ms": 26.13, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.89, + "duration_ms": 26.35, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.97, + "duration_ms": 25.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.08, + "duration_ms": 26.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.13, + "duration_ms": 25.98, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index c2b44a5..814050a 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 27.18 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 26.61 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 26.45 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.3 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 25.92 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 25.87 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.89 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.97 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.08 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.13 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 26.11 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 27.48 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 26.22 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.36 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 25.99 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.13 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.35 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.95 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.04 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.98 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index ba27e66..030f5a5 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,8 +8,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66", - "archive_entry_count": 513, + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_entry_count": 517, "failure_count": 0, "warning_count": 0 }, diff --git a/reports/package_verification.md b/reports/package_verification.md index 5835961..97731c0 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66` +- Archive SHA256: `4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 07d9a1f..da9dc69 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726", - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 52acab2..6958f1a 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726` -- Archive SHA256: `1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66` +- Package SHA256: `fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c` +- Archive SHA256: `4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a` - Install simulated: `True` ## Compatibility diff --git a/reports/review-studio.html b/reports/review-studio.html index 7f08768..2d6afcf 100644 --- a/reports/review-studio.html +++ b/reports/review-studio.html @@ -242,12 +242,12 @@

核心指标

-
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

76 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

513 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

+
Skill IR2.0.0

5 targets in platform-neutral contract

Compiler5/5

target contracts compiled from Skill IR

Output Delta100.0

5 cases; 1 file-backed

Exec Runs10

command 10; model 0; recorded 0

Blind A/B5

review pairs hide baseline vs with-skill labels

Review A/B0/5

adjudication decisions; pending 5

Runtime5/5

target conformance pass rate

Perm Probe4/4

0 native enforcement targets

Trust0

77 scripts scanned; secrets found

Atlas5

12 scanned skills; route collisions

Driftlow

1 metadata events; 0 missed triggers

Waivers0

0 gates covered; human risk decisions

Notes0/0

0 open blocker annotations

Registry1.1.0

5 targets; MIT license

Archivepass

517 zip entries; package verification

Installpass

4 adapters; 12 permissions enforced; 0 permission failures

Upgrademinor

declared minor; 0 breaking changes

审查闸门

-
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 76 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

reports/promotion_decisions.json + reports/upgrade_check.json + docs/migration-v2.md 证据
+
通过

意图画布

intent confidence 100/100; Intent is clear enough to package the first routeable version.

reports/intent-confidence.json 证据
通过

触发实验

13 trigger cases; 0 misroutes; 0 ambiguous

reports/route_scorecard.json 证据
关注

输出实验

5/5 cases; with-skill 100.0; baseline 0.0; file-backed 1; near-neighbor 1; blind A/B 5; exec 10; command 10; model 0; recorded 0; reviewed 0/5; review pending 5

reports/output_quality_scorecard.json 证据
通过

上下文

initial load 944/1000; quality density 137.7

reports/context_budget.json 证据
通过

运行矩阵

5 / 5 targets pass

reports/conformance_matrix.json 证据
通过

信任报告

0 secrets; 77 scripts; 3 network-capable scripts; 0 help smoke failures

reports/security_trust_report.json 证据
通过

权限批准

3/3 permissions approved; gaps 0; required file_write, network, subprocess

reports/security_trust_report.json + security/permission_policy.json 证据
通过

权限探针

4/4 targets probed; native 0; metadata fallback 4; residual risks 4

reports/runtime_permission_probes.json 证据
通过

组合治理

12 skills, 1 actionable; 0 actionable route collisions; 0 actionable owner gaps; 0 actionable stale; 0 actionable drift; 24 scoped non-actionable issues

reports/skill_atlas.json 证据
通过

运营回路

1 metadata events; adoption 0; missed 0; bad-output 0; risk low

reports/adoption_drift_report.json 证据
关注

人工批准

0 active waivers; 1 warning gates still need reviewer decision

reports/review_waivers.json 证据
通过

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

reports/registry_audit.json + reports/install_simulation.json 证据
通过

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

@@ -273,7 +273,7 @@

证据路径

- +
@@ -313,7 +313,7 @@
-

信任报告

Secret
0
脚本数
76
网络脚本
3
Help 失败
0
包体哈希
14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726
+

信任报告

Secret
0
脚本数
77
网络脚本
3
Help 失败
0
包体哈希
fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c

安全边界

高风险 secret、远程 inline execution、缺失依赖策略或无法解释的脚本接口应阻断 governed release。

@@ -344,12 +344,12 @@

注册审计

yao-meta-skill 1.1.0; 6/6 compatibility entries pass; install pass with 4 adapters; installer permissions 12 enforced / 0 failures

-

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66
+

包体元数据

名称
yao-meta-skill
版本
1.1.0
Maturity
governed
Owner
Yao Team
License
MIT
信任级别
local
目标平台
openai, claude, generic, agent-skills-compatible, vscode
兼容通过
6/6
归档哈希
4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a

发布路线

0 promote; 3 keep current; 0 blocked; upgrade minor declared / minor recommended

-

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
513
失败数
0
警告数
0
归档哈希
1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66
+

包体验证

目标数
4
Adapter
4
归档存在
是
Zip 条目
517
失败数
0
警告数
0
归档哈希
4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a
diff --git a/reports/review-studio.json b/reports/review-studio.json index 32c392d..b673f60 100644 --- a/reports/review-studio.json +++ b/reports/review-studio.json @@ -59,7 +59,7 @@ "key": "trust-report", "label": "信任报告", "status": "pass", - "detail": "0 secrets; 76 scripts; 3 network-capable scripts; 0 help smoke failures", + "detail": "0 secrets; 77 scripts; 3 network-capable scripts; 0 help smoke failures", "evidence": "reports/security_trust_report.json", "link": "security_trust_report.md" }, @@ -241,6 +241,7 @@ "output_blind_review": "reports/output_blind_review_pack.md", "output_review_decisions": "reports/output_review_decisions.json", "output_review_adjudication": "reports/output_review_adjudication.md", + "benchmark_reproducibility": "reports/benchmark_reproducibility.md", "runtime_conformance": "reports/conformance_matrix.md", "trust_report": "reports/security_trust_report.md", "permission_policy": "security/permission_policy.md", @@ -278,6 +279,7 @@ "reports/output_blind_review_pack.md", "reports/output_blind_answer_key.json", "reports/output_review_adjudication.md", + "reports/benchmark_reproducibility.md", "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/runtime_permission_probes.md", @@ -694,7 +696,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 76 + "file_count": 77 }, { "path": "evals", @@ -706,10 +708,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 177 + "file_count": 179 } ], - "file_count": 317, + "file_count": 320, "folder_count": 4, "distribution": [ { @@ -734,7 +736,7 @@ }, { "label": "scripts", - "value": 76 + "value": 77 }, { "label": "evals", @@ -742,7 +744,7 @@ }, { "label": "reports", - "value": 177 + "value": 179 } ] }, @@ -863,7 +865,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 76 + "file_count": 77 }, { "path": "evals", @@ -875,7 +877,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 177 + "file_count": 179 } ], "strengths": [ @@ -1142,6 +1144,36 @@ "reviewed_at": "", "failures": [] }, + "benchmark_reproducibility": { + "ok": true, + "summary": { + "reproducibility_ready": true, + "methodology_complete": true, + "required_artifact_count": 16, + "missing_artifact_count": 0, + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 15, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 0, + "token_observed_count": 0, + "human_review_complete": false, + "provider_evidence_complete": false, + "world_class_ready": false, + "world_class_open_gap_count": 4, + "world_class_task_count": 4, + "working_tree_dirty": true, + "changed_file_count": 43 + }, + "commit": "c2507af5267e3c1eb6802449f74236c5ab649742", + "missing_artifacts": [], + "limitations": [ + "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close." + ] + }, "runtime_conformance": { "target_count": 5, "pass_count": 5, @@ -1209,8 +1241,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 160, - "script_count": 76, + "scanned_files": 161, + "script_count": 77, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -1219,18 +1251,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 53, + "file_write_script_count": 54, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 66, + "help_smoke_checked_count": 67, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 160, - "package_sha256": "f328af72fb868e5d1d246d07b131a9372b30f781b3f887c306951cdf15a20315" + "package_hash_file_count": 161, + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" }, "skill_atlas": { "skill_count": 12, @@ -1268,8 +1300,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f328af72fb868e5d1d246d07b131a9372b30f781b3f887c306951cdf15a20315", - "archive_sha256": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, "compatibility": { "openai": "pass", @@ -1300,7 +1332,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1316,8 +1348,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f", - "archive_entry_count": 513, + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_entry_count": 517, "failure_count": 0, "warning_count": 0 }, @@ -1328,7 +1360,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 513, + "archive_entry_count": 517, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1395,12 +1427,12 @@ { "field": "archive_sha256", "from": "", - "to": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f" + "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f328af72fb868e5d1d246d07b131a9372b30f781b3f887c306951cdf15a20315" + "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" } ] }, @@ -3280,7 +3312,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.18, + "duration_ms": 26.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3308,7 +3340,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.61, + "duration_ms": 27.48, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3331,7 +3363,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.45, + "duration_ms": 26.22, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3359,7 +3391,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.3, + "duration_ms": 26.36, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3382,7 +3414,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.92, + "duration_ms": 25.99, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3410,7 +3442,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.87, + "duration_ms": 26.13, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3433,7 +3465,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.89, + "duration_ms": 26.35, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3460,7 +3492,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 25.97, + "duration_ms": 25.95, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3483,7 +3515,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.08, + "duration_ms": 26.04, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3512,7 +3544,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 26.13, + "duration_ms": 25.98, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -3994,6 +4026,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -4094,7 +4127,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -4200,7 +4233,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4222,6 +4255,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -4260,9 +4294,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -4291,15 +4326,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -4320,8 +4355,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -4341,7 +4376,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -4352,7 +4387,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -4433,7 +4468,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -4455,6 +4490,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -4493,9 +4529,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -4524,15 +4561,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -4551,8 +4588,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -4572,7 +4609,7 @@ "strategy": "Ship the neutral source tree and expose OpenAI-facing interface metadata as a generated companion file.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -4583,7 +4620,7 @@ }, "scripts": { "strategy": "Keep scripts as local package resources; expose help-smoke and permission metadata for reviewer approval before execution.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -4813,6 +4850,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -4913,7 +4951,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -5019,7 +5057,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5041,6 +5079,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -5079,9 +5118,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -5110,15 +5150,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -5139,8 +5179,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -5160,7 +5200,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -5171,7 +5211,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -5252,7 +5292,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5274,6 +5314,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -5312,9 +5353,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -5343,15 +5385,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -5370,8 +5412,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -5391,7 +5433,7 @@ "strategy": "Preserve the source tree directly; write target notes in targets/claude/README.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -5402,7 +5444,7 @@ }, "scripts": { "strategy": "Scripts remain local package resources and must be reviewed through trust and permission reports before use.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -5632,6 +5674,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -5732,7 +5775,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -5838,7 +5881,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -5860,6 +5903,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -5898,9 +5942,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -5929,15 +5974,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -5958,8 +6003,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -5979,7 +6024,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -5989,7 +6034,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -6064,7 +6109,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6086,6 +6131,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -6124,9 +6170,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -6155,15 +6202,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -6182,8 +6229,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -6203,7 +6250,7 @@ "strategy": "Preserve references, scripts, assets, evals, reports, and adapter metadata as relative package resources.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -6213,7 +6260,7 @@ }, "scripts": { "strategy": "Expose script and permission metadata for downstream clients or installers to enforce.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -6435,6 +6482,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -6535,7 +6583,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -6641,7 +6689,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6663,6 +6711,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -6701,9 +6750,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -6732,15 +6782,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -6761,8 +6811,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -6782,7 +6832,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -6793,7 +6843,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -6867,7 +6917,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -6889,6 +6939,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -6927,9 +6978,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -6958,15 +7010,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -6985,8 +7037,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -7006,7 +7058,7 @@ "strategy": "Keep optional directories as relative resources next to SKILL.md.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -7017,7 +7069,7 @@ }, "scripts": { "strategy": "Scripts remain local optional resources and should advertise --help when executable.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -7238,6 +7290,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -7338,7 +7391,7 @@ ], "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 } @@ -7444,7 +7497,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7466,6 +7519,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -7504,9 +7558,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -7535,15 +7590,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -7564,8 +7619,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -7585,7 +7640,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -7596,7 +7651,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -7674,7 +7729,7 @@ }, "file_write": { "required": true, - "script_count": 53, + "script_count": 54, "scripts": [ "scripts/adjudicate_output_review.py", "scripts/build_confusion_matrix.py", @@ -7696,6 +7751,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -7734,9 +7790,10 @@ }, "subprocess": { "required": true, - "script_count": 7, + "script_count": 8, "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -7765,15 +7822,15 @@ }, "help_smoke": { "enabled": true, - "checked_count": 66, + "checked_count": 67, "failed_count": 0, "failed_scripts": [] }, "trust_summary": { "secret_findings": 0, "network_script_count": 3, - "file_write_script_count": 53, - "subprocess_script_count": 7, + "file_write_script_count": 54, + "subprocess_script_count": 8, "interactive_script_count": 0, "help_smoke_failed_count": 0 } @@ -7792,8 +7849,8 @@ ], "capability_counts": { "network": 3, - "file_write": 53, - "subprocess": 7, + "file_write": 54, + "subprocess": 8, "interactive": 0 }, "evidence": "reports/security_trust_report.json", @@ -7813,7 +7870,7 @@ "strategy": "Install as project or user scoped skill source, preserving relative references and scripts.", "counts": { "references": 32, - "scripts": 76, + "scripts": 77, "assets": 2, "reports": 43 }, @@ -7824,7 +7881,7 @@ }, "scripts": { "strategy": "Scripts require workspace trust and operator/client approval outside this compiler.", - "script_count": 76, + "script_count": 77, "help_smoke_failed_count": 0 }, "permissions": { @@ -8464,8 +8521,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 160, - "script_count": 76, + "scanned_files": 161, + "script_count": 77, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -8474,18 +8531,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 53, + "file_write_script_count": 54, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 66, + "help_smoke_checked_count": 67, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 160, - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726" + "package_hash_file_count": 161, + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" }, "failures": [], "warnings": [], @@ -8896,6 +8953,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a release-facing benchmark reproducibility manifest and Markdown report.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_context_reports.py", "interface": "cli", @@ -9603,9 +9674,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 66, - "checked_count": 66, - "passed_count": 66, + "candidate_count": 67, + "checked_count": 67, + "passed_count": 67, "failed_count": 0, "skipped_count": 10, "failed_scripts": [], @@ -9890,6 +9961,16 @@ "stdout_excerpt": "usage: render_baseline_compare.py [-h] --entry ENTRY\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a baseline comparison report from description optimization ", "stderr_excerpt": "" }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "command": "python3 scripts/render_benchmark_reproducibility.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_benchmark_reproducibility.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_context_reports.py", "command": "python3 scripts/render_context_reports.py --help", @@ -10386,6 +10467,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -10442,6 +10524,7 @@ "status": "approved", "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", @@ -10485,10 +10568,10 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 751, - "other_text_tokens": 974109, + "other_text_tokens": 985553, "estimated_initial_load_tokens": 944, - "estimated_total_text_tokens": 974860, - "relevant_file_count": 425, + "estimated_total_text_tokens": 986304, + "relevant_file_count": 431, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 137.7 @@ -11380,6 +11463,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -12323,7 +12407,7 @@ "adoption_drift": { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-13T15:09:29Z", + "generated_at": "2026-06-13T00:00:00Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", @@ -12491,8 +12575,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726", - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, "compatibility": { "openai": "pass", @@ -12523,7 +12607,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -12548,7 +12632,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" } ] }, @@ -12571,8 +12655,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66", - "archive_entry_count": 513, + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_entry_count": 517, "failure_count": 0, "warning_count": 0 }, @@ -13237,7 +13321,7 @@ "installed_skill_dir": "[temporary-install-root]/yao-meta-skill", "summary": { "archive_present": true, - "archive_entry_count": 513, + "archive_entry_count": 517, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -13579,12 +13663,12 @@ { "field": "archive_sha256", "from": "", - "to": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66" + "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726" + "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" } ] }, diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index db739f5..6f6bc5d 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,8 +2,8 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 160, - "script_count": 76, + "scanned_files": 161, + "script_count": 77, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -12,18 +12,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 53, + "file_write_script_count": 54, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 66, + "help_smoke_checked_count": 67, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 160, - "package_sha256": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726" + "package_hash_file_count": 161, + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" }, "failures": [], "warnings": [], @@ -434,6 +434,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "interface": "cli", + "interface_declared": true, + "interface_reason": "Renders a release-facing benchmark reproducibility manifest and Markdown report.", + "has_argparse": true, + "has_main_guard": true, + "uses_input": false, + "uses_network": false, + "uses_file_write": true, + "uses_subprocess": true, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/render_context_reports.py", "interface": "cli", @@ -1141,9 +1155,9 @@ "help_smoke": { "enabled": true, "timeout_seconds": 5.0, - "candidate_count": 66, - "checked_count": 66, - "passed_count": 66, + "candidate_count": 67, + "checked_count": 67, + "passed_count": 67, "failed_count": 0, "skipped_count": 10, "failed_scripts": [], @@ -1428,6 +1442,16 @@ "stdout_excerpt": "usage: render_baseline_compare.py [-h] --entry ENTRY\n [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n\nRender a baseline comparison report from description optimization ", "stderr_excerpt": "" }, + { + "path": "scripts/render_benchmark_reproducibility.py", + "command": "python3 scripts/render_benchmark_reproducibility.py --help", + "returncode": 0, + "timed_out": false, + "passed": true, + "has_help_text": true, + "stdout_excerpt": "usage: render_benchmark_reproducibility.py [-h] [--output-json OUTPUT_JSON]\n [--output-md OUTPUT_MD]\n [--generated-at GENERATED_AT]\n ", + "stderr_excerpt": "" + }, { "path": "scripts/render_context_reports.py", "command": "python3 scripts/render_context_reports.py --help", @@ -1924,6 +1948,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", @@ -1980,6 +2005,7 @@ "status": "approved", "scripts": [ "scripts/ci_test.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_eval_dashboard.py", "scripts/run_eval_suite.py", "scripts/run_output_execution.py", diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 84c73d2..44cbfcd 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,22 +1,22 @@ # Security Trust Report - OK: `True` -- Scanned files: `160` -- Scripts: `76` +- Scanned files: `161` +- Scripts: `77` - Internal script modules: `10` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` - Network policy missing scripts: `0` -- File-write scripts: `53` +- File-write scripts: `54` - Permission approvals: `3 / 3` - Permission approval gaps: `0` -- CLI help smoke checked: `66` +- CLI help smoke checked: `67` - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `160` -- Package SHA256: `14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726` +- Package hash files: `161` +- Package SHA256: `fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c` ## Failures @@ -54,8 +54,8 @@ - Enabled: `True` - Timeout seconds: `5.0` -- Checked scripts: `66` -- Passed scripts: `66` +- Checked scripts: `67` +- Passed scripts: `67` - Failed scripts: `none` ## Script Surface @@ -90,6 +90,7 @@ | scripts/render_adoption_drift_report.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_artifact_design_profile.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_baseline_compare.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | +| scripts/render_benchmark_reproducibility.py | cli | True | True | True | False | False | True | True | Renders a release-facing benchmark reproducibility manifest and Markdown report. | | scripts/render_context_reports.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_description_drift_history.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/render_eval_dashboard.py | cli | False | True | True | False | False | True | True | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 65e05ee..55eacd9 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -15,6 +15,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - Bilingual Skill Overview v2 that includes these evidence surfaces. - Skill OS 2.0 Audit v0 for requirement-by-requirement evidence mapping, with explicit separation between local evidence, human-required review, and external-required provider/client integrations. - World-Class Evidence Plan v0 that turns the remaining provider holdout, human adjudication, native permission enforcement, and native client telemetry gaps into executable runbooks with success checks and privacy contracts. +- Benchmark Reproducibility v0 that turns public benchmark methodology, required artifacts, failure disclosure, and reproduction commands into machine-checkable release evidence. - Review Studio 2.0 v0 for one-page blocker, warning, evidence-path, review-action, and release-gate review. - Review Studio Source Refs v0 so every non-pass review action can expose structured relative source/report links with best-effort line numbers. - Target Compiler v0 so Skill IR compiles into OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot target contracts before packaging, including target-specific permission contracts and native behavior contracts. @@ -50,6 +51,7 @@ This is still not the final world-class state. Target-native behavior contracts | Target Compiler | `scripts/compile_skill.py`, `reports/compiled_targets.md`, `tests/verify_compile_skill.py` | v0 landed | | Output Eval Lab | `evals/output/cases.jsonl`, `scripts/run_output_eval.py`, `scripts/run_output_execution.py`, `scripts/local_output_eval_runner.py`, `scripts/provider_output_eval_runner.py`, `scripts/adjudicate_output_review.py`, `reports/output_quality_scorecard.md`, `reports/output_execution_runs.md`, `reports/output_blind_review_pack.md`, `reports/output_blind_answer_key.json`, `reports/output_review_decisions.json`, `reports/output_review_adjudication.md` | v0 landed | | Benchmark methodology | `reports/benchmark_methodology.md` | v0 landed | +| Benchmark Reproducibility | `scripts/render_benchmark_reproducibility.py`, `reports/benchmark_reproducibility.md`, `tests/verify_benchmark_reproducibility.py` | v0 landed | | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | @@ -110,24 +112,25 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `66 / 66` CLI help smoke checks passing across `76` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `10` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `67 / 67` CLI help smoke checks passing across `77` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | | Registry Audit | package metadata generated with version, owner, license, source checksum, archive checksum, Skill IR provenance, and compatibility matrix | -| Package Verification | `4 / 4` target adapters present, archive verified, `513` zip entries, `0` failures, `0` warnings | -| Install Simulation | archive with `513` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | +| Package Verification | `4 / 4` target adapters present, archive verified, `517` zip entries, `0` failures, `0` warnings | +| Install Simulation | archive with `517` entries extracted into a local verification root, entrypoint/manifest/interface loaded, reports present, `4` adapters readable, `12` installer permission checks enforced, `0` permission failures, `0` failures, `0` warnings | | Local Install Sync Preflight | `make sync-local-install` and `make sync-active-install` rebuild the package first, then sync only after install simulation passes with `12` enforced installer permission checks and `0` permission failures | | Upgrade Check | current package declares `minor` over the 1.0.0 baseline, recommended bump is `minor`, and release notes include added targets plus checksum changes | | Adoption Drift | `1` metadata-only review event, `0` adoption samples, adoption `0`, risk band `low`; optional `yao.py` CLI capture, external client `telemetry-emit`, `5` `telemetry-hooks` recipes, Browser/Chrome native messaging host, and validated external JSONL import are available but off by default for reproducible release evidence; raw `reports/telemetry_events.jsonl` is gitignored and blocked from zip packages | | Review Waivers | ledger generated; current release has `1` warning gate that still needs reviewer decision or a time-bounded waiver; blockers remain non-waivable in v0 | | Review Annotations | ledger generated; current release has `0` reviewer annotations and `0` open annotation blockers | | Review Studio | decision `review`, world-class score `92`, `13` gates, `0` blockers, `2` warnings, `2` review actions, `0` open annotation blockers | -| Skill OS 2.0 Audit | `14` audited areas, with local foundation evidence separated from human-required and external-required gaps | +| Skill OS 2.0 Audit | `15` audited areas, with local foundation evidence separated from human-required and external-required gaps | | World-Class Evidence Plan | `4` remaining evidence tasks: `1` human-required and `3` external-required, all with runbooks, success checks, and privacy contracts | +| Benchmark Reproducibility | local reproducibility ready with `16` required artifacts, `0` missing artifacts, `15` reproduction commands, and `3` disclosed failure cases; provider and human evidence remain explicit limitations | | IR-first Packaging | `openai`, `claude`, `generic`, and `vscode` adapters include compiler contracts, permission contracts, target-native behavior contracts, IR provenance, semantic parity checks, and install-scope notes where applicable | | Context Budget | initial load `944/1000`, under the production budget | -| CI | `make ci-test` target count is `62` after the World-Class Evidence Plan gate update | +| CI | `make ci-test` target count is `63` after the Benchmark Reproducibility gate update | ## Next Highest-Leverage Moves diff --git a/reports/skill-overview.html b/reports/skill-overview.html index 405d949..65ee8dc 100644 --- a/reports/skill-overview.html +++ b/reports/skill-overview.html @@ -621,7 +621,7 @@

作用定位Role

-
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
+
  • 从工作流、提示词、对话记录、文档或笔记中创建、重构、评估和打包 agent skill；适用于新建 Skill、沉淀重复流程、改进现有 Skill、补充 eval 或团队复用打包。Create, refactor, evaluate, and package agent skills from workflows, prompts, transcripts, docs, or notes. Use when asked to create a skill, turn a repeated process into a reusable skill, improve an existing skill, add evals, or package a skill for team reuse.
  • 把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。把一次性经验沉淀为可复用、可评估、可迁移的 Skill 包体。
  • 交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html交付结果：SKILL.md, agents/interface.yaml, reports/skill-ir.json, reports/compiled_targets.md, reports/output_quality_scorecard.md, reports/output_execution_runs.md, reports/output_blind_review_pack.md, reports/output_blind_answer_key.json, reports/output_review_adjudication.md, reports/benchmark_reproducibility.md, reports/conformance_matrix.md, reports/security_trust_report.md, reports/runtime_permission_probes.md, reports/skill_atlas.html, reports/registry_audit.md, reports/package_verification.md, reports/install_simulation.md, reports/upgrade_check.md, reports/adoption_drift_report.md, reports/review_waivers.md, reports/world_class_evidence_plan.md, reports/review_annotations.md, reports/review-studio.html, reports/skill-overview.html
交付流程输入材料Skill 包体可复用能力
交付流程把用户输入、生成的包体和可复用能力放在一条线上。
@@ -771,7 +771,7 @@

让 reviewer 快速确认关键文件、目录和资产分布。Lets reviewers confirm key files, directories, and asset distribution quickly.

-
资产分布317项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
+
资产分布320项SKILL.mdREADME.mdagents/interface.yamlmanifest.jsonreferencesscripts
资产分布图展示当前包体的文件和目录重心。
diff --git a/reports/skill-overview.json b/reports/skill-overview.json index d33afe2..57073bd 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -18,6 +18,7 @@ "reports/output_blind_review_pack.md", "reports/output_blind_answer_key.json", "reports/output_review_adjudication.md", + "reports/benchmark_reproducibility.md", "reports/conformance_matrix.md", "reports/security_trust_report.md", "reports/runtime_permission_probes.md", @@ -434,7 +435,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 76 + "file_count": 77 }, { "path": "evals", @@ -446,10 +447,10 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 177 + "file_count": 179 } ], - "file_count": 317, + "file_count": 320, "folder_count": 4, "distribution": [ { @@ -474,7 +475,7 @@ }, { "label": "scripts", - "value": 76 + "value": 77 }, { "label": "evals", @@ -482,7 +483,7 @@ }, { "label": "reports", - "value": 177 + "value": 179 } ] }, @@ -603,7 +604,7 @@ "path": "scripts", "label": "Deterministic helpers or local tooling", "kind": "folder", - "file_count": 76 + "file_count": 77 }, { "path": "evals", @@ -615,7 +616,7 @@ "path": "reports", "label": "Generated evidence and overview artifacts", "kind": "folder", - "file_count": 177 + "file_count": 179 } ], "strengths": [ @@ -882,6 +883,36 @@ "reviewed_at": "", "failures": [] }, + "benchmark_reproducibility": { + "ok": true, + "summary": { + "reproducibility_ready": true, + "methodology_complete": true, + "required_artifact_count": 16, + "missing_artifact_count": 0, + "output_case_count": 5, + "failure_disclosure_count": 3, + "command_count": 15, + "command_executed_count": 10, + "timing_observed_count": 10, + "model_executed_count": 0, + "token_observed_count": 0, + "human_review_complete": false, + "provider_evidence_complete": false, + "world_class_ready": false, + "world_class_open_gap_count": 4, + "world_class_task_count": 4, + "working_tree_dirty": true, + "changed_file_count": 43 + }, + "commit": "c2507af5267e3c1eb6802449f74236c5ab649742", + "missing_artifacts": [], + "limitations": [ + "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close." + ] + }, "runtime_conformance": { "target_count": 5, "pass_count": 5, @@ -949,8 +980,8 @@ "failures": [] }, "trust_security": { - "scanned_files": 160, - "script_count": 76, + "scanned_files": 161, + "script_count": 77, "internal_module_count": 10, "secret_findings": 0, "dependency_files": [ @@ -959,18 +990,18 @@ "network_script_count": 3, "network_policy_covered_count": 3, "network_policy_missing_count": 0, - "file_write_script_count": 53, + "file_write_script_count": 54, "permission_required_count": 3, "permission_approved_count": 3, "permission_missing_count": 0, "permission_invalid_count": 0, "permission_expired_count": 0, - "help_smoke_checked_count": 66, + "help_smoke_checked_count": 67, "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 160, - "package_sha256": "f328af72fb868e5d1d246d07b131a9372b30f781b3f887c306951cdf15a20315" + "package_hash_file_count": 161, + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" }, "skill_atlas": { "skill_count": 12, @@ -1008,8 +1039,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "f328af72fb868e5d1d246d07b131a9372b30f781b3f887c306951cdf15a20315", - "archive_sha256": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f" + "package_sha256": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, "compatibility": { "openai": "pass", @@ -1040,7 +1071,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f", + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -1056,8 +1087,8 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f", - "archive_entry_count": 513, + "archive_sha256": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a", + "archive_entry_count": 517, "failure_count": 0, "warning_count": 0 }, @@ -1068,7 +1099,7 @@ "ok": true, "summary": { "archive_present": true, - "archive_entry_count": 513, + "archive_entry_count": 517, "archive_extracted": true, "entrypoint_loaded": true, "manifest_loaded": true, @@ -1135,12 +1166,12 @@ { "field": "archive_sha256", "from": "", - "to": "665b58c8a8b1ea1983a50c160e28453d7a2255ac17d4b56bb7d31f2d89a3850f" + "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "f328af72fb868e5d1d246d07b131a9372b30f781b3f887c306951cdf15a20315" + "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" } ] }, diff --git a/reports/skill_atlas.json b/reports/skill_atlas.json index 5eff35b..14931ef 100644 --- a/reports/skill_atlas.json +++ b/reports/skill_atlas.json @@ -92,6 +92,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 2a51a6f..d50b795 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -4,8 +4,8 @@ "generated_at": "2026-06-13", "skill_dir": ".", "summary": { - "item_count": 14, - "pass_count": 10, + "item_count": 15, + "pass_count": 11, "warn_count": 0, "human_required_count": 1, "external_required_count": 3, @@ -16,7 +16,7 @@ "decision": "continue-iteration" }, "status_counts": { - "pass": 10, + "pass": 11, "warn": 0, "human_required": 1, "external_required": 3, @@ -141,6 +141,36 @@ ], "next_action": "Record real A/B choices in the decision template, then regenerate adjudication." }, + { + "key": "benchmark-reproducibility", + "label": "Benchmark Reproducibility", + "status": "pass", + "current": "artifacts 16; missing 0; failures 3", + "target": "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable", + "evidence": [ + { + "path": "reports/benchmark_methodology.md", + "exists": true + }, + { + "path": "reports/benchmark_reproducibility.json", + "exists": true + }, + { + "path": "reports/benchmark_reproducibility.md", + "exists": true + }, + { + "path": "evals/failure-cases.md", + "exists": true + }, + { + "path": "tests/verify_benchmark_reproducibility.py", + "exists": true + } + ], + "next_action": "Keep the manifest current with every benchmark, package, and release evidence change." + }, { "key": "runtime-conformance", "label": "Runtime Conformance", @@ -167,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 76; help failures 0", + "current": "secrets 0; scripts 77; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { @@ -251,7 +281,7 @@ "key": "registry-distribution", "label": "Registry Distribution", "status": "pass", - "current": "zip entries 513; install failures 0; permission failures 0", + "current": "zip entries 517; install failures 0; permission failures 0", "target": "Package metadata, archive checksum, package verification, and install simulation pass", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 0a195d3..0c07a94 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -5,7 +5,7 @@ Generated at: `2026-06-13` ## Summary - decision: `continue-iteration` -- pass: `10` / `14` +- pass: `11` / `15` - human required: `1` - external required: `3` - missing: `0` @@ -21,12 +21,13 @@ Generated at: `2026-06-13` | Output Eval Lab | pass | 5 cases; delta 100.0; exec 10; blind 5 | with-skill/baseline, assertions, execution evidence, blind A/B, failure taxonomy | Add more real-file and adversarial holdout cases as usage grows. | | Provider Holdout | external-required | model-executed 0; token-observed 0 | At least one real provider-backed holdout run with observed model/timing/token metadata | Run provider-backed holdout cases with real credentials and commit only aggregate evidence. | | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | +| Benchmark Reproducibility | pass | artifacts 16; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 76; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 77; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4 | Packaged adapters expose explicit permission metadata and residual risks | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0 | At least one target/client enforces approved permissions at runtime | Integrate a real client or installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | -| Registry Distribution | pass | zip entries 513; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | +| Registry Distribution | pass | zip entries 517; install failures 0; permission failures 0 | Package metadata, archive checksum, package verification, and install simulation pass | Regenerate registry after package verification so checksums stay aligned. | | Review Studio | pass | decision review; warnings 2; score 92 | One page shows gates, evidence paths, blockers, warnings, actions, waivers, and annotations | Resolve human/external warning gates before claiming full release readiness. | | Telemetry Drift | pass | events 1; risk low; recipes 5 | Local-first metadata-only event contract, aggregate drift report, hook recipes, and import path | Keep raw JSONL out of distributed packages and use aggregate reports for Atlas. | | Native Client Telemetry | external-required | external source events 0; adoption samples 0 | A real Browser/Chrome/provider client sends production metadata events | Install a real client against the native host and import production metadata-only events. | @@ -60,6 +61,10 @@ Generated at: `2026-06-13` - existing evidence: `reports/output_review_decisions.json`, `reports/output_review_adjudication.json`, `scripts/adjudicate_output_review.py` +### Benchmark Reproducibility + +- existing evidence: `reports/benchmark_methodology.md`, `reports/benchmark_reproducibility.json`, `reports/benchmark_reproducibility.md`, `evals/failure-cases.md`, `tests/verify_benchmark_reproducibility.py` + ### Runtime Conformance - existing evidence: `runtime/conformance/schema.json`, `scripts/run_conformance_suite.py`, `reports/conformance_matrix.json` diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index 55a68c1..ffb6bbd 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "1220ca743cc885c1818fbd10b0e24dc9b451a932ed51282e1f801eb598526a66" + "to": "4d87b1f2b91c08d6d2110342dff3ec7c46d9f58f652abace97270c708191a86a" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "14948119733df8546f5c30e0131428ea95c4866a3bdc8151a3e1ab9c7a4f1726" + "to": "fcc89390653cca0e31cb56be360951474028409c160fb0d975b361aad557e96c" } ] }, diff --git a/scripts/ci_test.py b/scripts/ci_test.py index e477e94..4dc0f50 100644 --- a/scripts/ci_test.py +++ b/scripts/ci_test.py @@ -50,6 +50,7 @@ DEFAULT_TARGETS = [ "review-studio-check", "skill-os2-audit-check", "world-class-evidence-check", + "benchmark-reproducibility-check", "feedback-check", "adoption-drift-check", "telemetry-import-check", diff --git a/scripts/render_benchmark_reproducibility.py b/scripts/render_benchmark_reproducibility.py new file mode 100644 index 0000000..b0839eb --- /dev/null +++ b/scripts/render_benchmark_reproducibility.py @@ -0,0 +1,368 @@ +#!/usr/bin/env python3 +import argparse +import hashlib +import json +import subprocess +from datetime import date +from pathlib import Path +from typing import Any + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT_INTERFACE = "cli" +SCRIPT_INTERFACE_REASON = "Renders a release-facing benchmark reproducibility manifest and Markdown report." + +METHODOLOGY_SECTIONS = [ + "## Benchmark Types", + "## Sample Sources", + "## Evaluation Dimensions", + "## Weighting Rule", + "## Failure Disclosure", + "## Reproduction", +] + +REQUIRED_ARTIFACTS = [ + ("methodology", "reports/benchmark_methodology.md"), + ("failure_disclosure", "evals/failure-cases.md"), + ("output_cases", "evals/output/cases.jsonl"), + ("output_schema", "evals/output/schema.json"), + ("output_scorecard", "reports/output_quality_scorecard.json"), + ("output_execution", "reports/output_execution_runs.json"), + ("blind_review", "reports/output_blind_review_pack.json"), + ("review_adjudication", "reports/output_review_adjudication.json"), + ("trigger_scorecard", "reports/route_scorecard.json"), + ("runtime_conformance", "reports/conformance_matrix.json"), + ("trust_report", "reports/security_trust_report.json"), + ("registry_audit", "reports/registry_audit.json"), + ("package_verification", "reports/package_verification.json"), + ("install_simulation", "reports/install_simulation.json"), + ("skill_os2_audit", "reports/skill_os2_audit.json"), + ("world_class_evidence_plan", "reports/world_class_evidence_plan.json"), +] + +REPRODUCTION_COMMANDS = [ + { + "label": "source commit", + "command": "git rev-parse HEAD", + "evidence": "git commit hash", + }, + { + "label": "trigger eval", + "command": "make eval-suite", + "evidence": "reports/eval_suite.json", + }, + { + "label": "output eval", + "command": "python3 scripts/yao.py output-eval", + "evidence": "reports/output_quality_scorecard.json", + }, + { + "label": "output execution", + "command": "python3 scripts/yao.py output-exec --runner-command '[\"python3\",\"scripts/local_output_eval_runner.py\"]'", + "evidence": "reports/output_execution_runs.json", + }, + { + "label": "blind review adjudication", + "command": "python3 scripts/yao.py output-review", + "evidence": "reports/output_review_adjudication.json", + }, + { + "label": "skill ir", + "command": "python3 scripts/yao.py skill-ir . --output-json skill-ir/examples/yao-meta-skill.json", + "evidence": "skill-ir/examples/yao-meta-skill.json", + }, + { + "label": "runtime conformance", + "command": "python3 scripts/yao.py conformance .", + "evidence": "reports/conformance_matrix.json", + }, + { + "label": "trust report", + "command": "python3 scripts/yao.py trust .", + "evidence": "reports/security_trust_report.json", + }, + { + "label": "package", + "command": "python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip", + "evidence": "dist/yao-meta-skill.zip", + }, + { + "label": "package verify", + "command": "python3 scripts/yao.py package-verify . --package-dir dist --require-zip", + "evidence": "reports/package_verification.json", + }, + { + "label": "install simulate", + "command": "python3 scripts/yao.py install-simulate . --package-dir dist", + "evidence": "reports/install_simulation.json", + }, + { + "label": "registry audit", + "command": "python3 scripts/yao.py registry-audit .", + "evidence": "reports/registry_audit.json", + }, + { + "label": "skill os audit", + "command": "python3 scripts/yao.py skill-os2-audit .", + "evidence": "reports/skill_os2_audit.json", + }, + { + "label": "world-class evidence plan", + "command": "python3 scripts/yao.py world-class-evidence .", + "evidence": "reports/world_class_evidence_plan.json", + }, + { + "label": "full ci", + "command": "make ci-test", + "evidence": "CI target output", + }, +] + + +def load_json(path: Path) -> dict[str, Any]: + if not path.exists(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except json.JSONDecodeError: + return {} + return payload if isinstance(payload, dict) else {} + + +def rel_path(path: Path, root: Path) -> str: + try: + return str(path.resolve().relative_to(root.resolve())) + except ValueError: + return str(path.resolve()) + + +def sha256_file(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def git_commit(skill_dir: Path) -> str: + try: + proc = subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=skill_dir, + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return "unknown" + return proc.stdout.strip() or "unknown" + + +def git_status(skill_dir: Path) -> dict[str, Any]: + try: + proc = subprocess.run( + ["git", "status", "--short", "--untracked-files=all"], + cwd=skill_dir, + capture_output=True, + text=True, + check=True, + ) + except (OSError, subprocess.CalledProcessError): + return {"available": False, "dirty": None, "changed_file_count": None} + lines = [line for line in proc.stdout.splitlines() if line.strip()] + return { + "available": True, + "dirty": bool(lines), + "changed_file_count": len(lines), + "sample": lines[:12], + } + + +def count_jsonl(path: Path) -> int: + if not path.exists(): + return 0 + return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.strip()) + + +def count_failure_cases(path: Path) -> int: + if not path.exists(): + return 0 + return sum(1 for line in path.read_text(encoding="utf-8").splitlines() if line.startswith("### ")) + + +def methodology_check(path: Path) -> dict[str, Any]: + text = path.read_text(encoding="utf-8") if path.exists() else "" + sections = [{"heading": heading, "exists": heading in text} for heading in METHODOLOGY_SECTIONS] + return { + "path": "reports/benchmark_methodology.md", + "exists": path.exists(), + "sections": sections, + "missing_sections": [item["heading"] for item in sections if not item["exists"]], + } + + +def artifact_record(skill_dir: Path, label: str, rel: str) -> dict[str, Any]: + path = skill_dir / rel + record: dict[str, Any] = { + "label": label, + "path": rel, + "exists": path.exists(), + } + if path.exists() and path.is_file(): + record["bytes"] = path.stat().st_size + record["sha256"] = sha256_file(path) + return record + + +def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]: + reports = skill_dir / "reports" + output_quality = load_json(reports / "output_quality_scorecard.json") + output_execution = load_json(reports / "output_execution_runs.json") + output_review = load_json(reports / "output_review_adjudication.json") + skill_os2 = load_json(reports / "skill_os2_audit.json") + world_class_plan = load_json(reports / "world_class_evidence_plan.json") + methodology = methodology_check(reports / "benchmark_methodology.md") + artifacts = [artifact_record(skill_dir, label, rel) for label, rel in REQUIRED_ARTIFACTS] + missing_artifacts = [item["path"] for item in artifacts if not item["exists"]] + output_summary = output_quality.get("summary", {}) + execution_summary = output_execution.get("summary", {}) + review_summary = output_review.get("summary", {}) + failure_case_count = count_failure_cases(skill_dir / "evals" / "failure-cases.md") + output_case_count = count_jsonl(skill_dir / "evals" / "output" / "cases.jsonl") + status = git_status(skill_dir) + local_reproducibility_ready = ( + not methodology["missing_sections"] + and not missing_artifacts + and output_case_count >= 5 + and failure_case_count > 0 + and output_summary.get("gate_pass") is True + and execution_summary.get("command_executed_count", 0) > 0 + and execution_summary.get("timing_observed_count", 0) > 0 + ) + human_review_complete = review_summary.get("pair_count", 0) > 0 and review_summary.get("pending_count", 0) == 0 + provider_evidence_complete = execution_summary.get("model_executed_count", 0) > 0 and execution_summary.get("token_observed_count", 0) > 0 + return { + "schema_version": "1.0", + "ok": local_reproducibility_ready, + "generated_at": generated_at, + "skill_dir": rel_path(skill_dir, ROOT), + "commit": git_commit(skill_dir), + "git_status": status, + "summary": { + "reproducibility_ready": local_reproducibility_ready, + "methodology_complete": not methodology["missing_sections"], + "required_artifact_count": len(artifacts), + "missing_artifact_count": len(missing_artifacts), + "output_case_count": output_case_count, + "failure_disclosure_count": failure_case_count, + "command_count": len(REPRODUCTION_COMMANDS), + "command_executed_count": execution_summary.get("command_executed_count", 0), + "timing_observed_count": execution_summary.get("timing_observed_count", 0), + "model_executed_count": execution_summary.get("model_executed_count", 0), + "token_observed_count": execution_summary.get("token_observed_count", 0), + "human_review_complete": human_review_complete, + "provider_evidence_complete": provider_evidence_complete, + "world_class_ready": bool(skill_os2.get("summary", {}).get("world_class_ready", False)), + "world_class_open_gap_count": skill_os2.get("summary", {}).get("open_gap_count", 0), + "world_class_task_count": world_class_plan.get("summary", {}).get("task_count", 0), + "working_tree_dirty": status.get("dirty"), + "changed_file_count": status.get("changed_file_count"), + }, + "methodology": methodology, + "artifacts_checked": artifacts, + "missing_artifacts": missing_artifacts, + "reproduction_commands": REPRODUCTION_COMMANDS, + "failure_disclosure": { + "path": "evals/failure-cases.md", + "case_count": failure_case_count, + "policy": "Keep representative failures visible and tied to regression checks.", + }, + "limitations": [ + "Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.", + "Pending blind-review decisions are visible but do not count as human adjudication.", + "World-class readiness remains false until external and human evidence gaps close.", + ], + "artifacts": { + "json": "reports/benchmark_reproducibility.json", + "markdown": "reports/benchmark_reproducibility.md", + }, + } + + +def render_markdown(report: dict[str, Any]) -> str: + summary = report["summary"] + lines = [ + "# Benchmark Reproducibility", + "", + f"Generated at: `{report['generated_at']}`", + f"Commit: `{report['commit']}`", + f"Working tree dirty at generation: `{str(summary.get('working_tree_dirty')).lower()}`", + "", + "## Summary", + "", + f"- reproducibility ready: `{str(summary['reproducibility_ready']).lower()}`", + f"- methodology complete: `{str(summary['methodology_complete']).lower()}`", + f"- required artifacts: `{summary['required_artifact_count']}`", + f"- missing artifacts: `{summary['missing_artifact_count']}`", + f"- output cases: `{summary['output_case_count']}`", + f"- disclosed failure cases: `{summary['failure_disclosure_count']}`", + f"- reproduction commands: `{summary['command_count']}`", + f"- provider evidence complete: `{str(summary['provider_evidence_complete']).lower()}`", + f"- human review complete: `{str(summary['human_review_complete']).lower()}`", + f"- world-class ready: `{str(summary['world_class_ready']).lower()}`", + f"- changed files at generation: `{summary.get('changed_file_count')}`", + "", + "This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete.", + "", + "## Methodology Sections", + "", + "| Section | Status |", + "| --- | --- |", + ] + for section in report["methodology"]["sections"]: + lines.append(f"| `{section['heading']}` | {'present' if section['exists'] else 'missing'} |") + lines.extend(["", "## Required Artifacts", "", "| Label | Path | Status | SHA256 |", "| --- | --- | --- | --- |"]) + for artifact in report["artifacts_checked"]: + digest = artifact.get("sha256", "") + lines.append( + f"| {artifact['label']} | `{artifact['path']}` | {'present' if artifact['exists'] else 'missing'} | `{digest[:12]}` |" + ) + lines.extend(["", "## Reproduction Commands", ""]) + for command in report["reproduction_commands"]: + lines.append(f"- `{command['command']}`") + lines.append(f" - evidence: `{command['evidence']}`") + lines.extend(["", "## Failure Disclosure", ""]) + disclosure = report["failure_disclosure"] + lines.append(f"- path: `{disclosure['path']}`") + lines.append(f"- disclosed cases: `{disclosure['case_count']}`") + lines.append(f"- policy: {disclosure['policy']}") + lines.extend(["", "## Limits", ""]) + lines.extend(f"- {item}" for item in report["limitations"]) + return "\n".join(lines).rstrip() + "\n" + + +def main() -> None: + parser = argparse.ArgumentParser(description="Render benchmark reproducibility evidence.") + parser.add_argument("skill_dir", nargs="?", default=".") + parser.add_argument("--output-json", default="reports/benchmark_reproducibility.json") + parser.add_argument("--output-md", default="reports/benchmark_reproducibility.md") + parser.add_argument("--generated-at", default=date.today().isoformat()) + args = parser.parse_args() + + skill_dir = Path(args.skill_dir).resolve() + report = build_report(skill_dir, args.generated_at) + output_json = Path(args.output_json) + output_md = Path(args.output_md) + if not output_json.is_absolute(): + output_json = skill_dir / output_json + if not output_md.is_absolute(): + output_md = skill_dir / output_md + output_json.parent.mkdir(parents=True, exist_ok=True) + output_md.parent.mkdir(parents=True, exist_ok=True) + output_json.write_text(json.dumps(report, ensure_ascii=False, indent=2) + "\n", encoding="utf-8") + output_md.write_text(render_markdown(report), encoding="utf-8") + print(json.dumps(report, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/scripts/render_review_studio.py b/scripts/render_review_studio.py index 726d711..e1ea250 100644 --- a/scripts/render_review_studio.py +++ b/scripts/render_review_studio.py @@ -124,6 +124,7 @@ def evidence_paths(skill_dir: Path) -> dict[str, str]: "output_blind_review": "reports/output_blind_review_pack.md", "output_review_decisions": "reports/output_review_decisions.json", "output_review_adjudication": "reports/output_review_adjudication.md", + "benchmark_reproducibility": "reports/benchmark_reproducibility.md", "runtime_conformance": "reports/conformance_matrix.md", "trust_report": "reports/security_trust_report.md", "permission_policy": "security/permission_policy.md", diff --git a/scripts/render_skill_os2_audit.py b/scripts/render_skill_os2_audit.py index eba69fe..7b9755d 100644 --- a/scripts/render_skill_os2_audit.py +++ b/scripts/render_skill_os2_audit.py @@ -77,6 +77,7 @@ def build_audit(skill_dir: Path, generated_at: str) -> dict[str, Any]: output_quality = load_json(reports / "output_quality_scorecard.json") output_execution = load_json(reports / "output_execution_runs.json") output_review = load_json(reports / "output_review_adjudication.json") + benchmark_reproducibility = load_json(reports / "benchmark_reproducibility.json") conformance = load_json(reports / "conformance_matrix.json") trust = load_json(reports / "security_trust_report.json") runtime_permissions = load_json(reports / "runtime_permission_probes.json") @@ -92,6 +93,7 @@ def build_audit(skill_dir: Path, generated_at: str) -> dict[str, Any]: output_summary = output_quality.get("summary", {}) execution_summary = output_execution.get("summary", {}) review_summary = output_review.get("summary", {}) + benchmark_summary = benchmark_reproducibility.get("summary", {}) conformance_summary = conformance.get("summary", {}) trust_summary = trust.get("summary", {}) permission_summary = runtime_permissions.get("summary", {}) @@ -171,6 +173,31 @@ def build_audit(skill_dir: Path, generated_at: str) -> dict[str, Any]: evidence(skill_dir, "reports/output_review_decisions.json", "reports/output_review_adjudication.json", "scripts/adjudicate_output_review.py"), "Record real A/B choices in the decision template, then regenerate adjudication.", ), + audit_item( + "benchmark-reproducibility", + "Benchmark Reproducibility", + status_from( + benchmark_reproducibility.get("ok") is True + and benchmark_summary.get("reproducibility_ready") is True + and benchmark_summary.get("failure_disclosure_count", 0) > 0 + and benchmark_summary.get("missing_artifact_count", 1) == 0 + ), + ( + f"artifacts {benchmark_summary.get('required_artifact_count', 0)}; " + f"missing {benchmark_summary.get('missing_artifact_count', 'n/a')}; " + f"failures {benchmark_summary.get('failure_disclosure_count', 0)}" + ), + "Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable", + evidence( + skill_dir, + "reports/benchmark_methodology.md", + "reports/benchmark_reproducibility.json", + "reports/benchmark_reproducibility.md", + "evals/failure-cases.md", + "tests/verify_benchmark_reproducibility.py", + ), + "Keep the manifest current with every benchmark, package, and release evidence change.", + ), audit_item( "runtime-conformance", "Runtime Conformance", diff --git a/scripts/skill_report_model.py b/scripts/skill_report_model.py index 849ac7c..13606ec 100644 --- a/scripts/skill_report_model.py +++ b/scripts/skill_report_model.py @@ -476,6 +476,7 @@ def build_report_model(skill_dir: Path) -> dict: output_execution = load_json(skill_dir / "reports" / "output_execution_runs.json") output_blind_review = load_json(skill_dir / "reports" / "output_blind_review_pack.json") output_review_adjudication = load_json(skill_dir / "reports" / "output_review_adjudication.json") + benchmark_reproducibility = load_json(skill_dir / "reports" / "benchmark_reproducibility.json") conformance = load_json(skill_dir / "reports" / "conformance_matrix.json") runtime_permissions = load_json(skill_dir / "reports" / "runtime_permission_probes.json") trust_report = load_json(skill_dir / "reports" / "security_trust_report.json") @@ -551,6 +552,9 @@ def build_report_model(skill_dir: Path) -> dict: if (skill_dir / "reports" / "output_review_adjudication.md").exists(): insert_after = deliverables.index("reports/output_blind_answer_key.json") + 1 if "reports/output_blind_answer_key.json" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 deliverables.insert(insert_after, "reports/output_review_adjudication.md") + if (skill_dir / "reports" / "benchmark_reproducibility.md").exists(): + insert_after = deliverables.index("reports/output_review_adjudication.md") + 1 if "reports/output_review_adjudication.md" in deliverables else deliverables.index("reports/output_quality_scorecard.md") + 1 + deliverables.insert(insert_after, "reports/benchmark_reproducibility.md") if (skill_dir / "reports" / "world_class_evidence_plan.md").exists(): insert_after = deliverables.index("reports/review_waivers.md") + 1 deliverables.insert(insert_after, "reports/world_class_evidence_plan.md") @@ -675,6 +679,13 @@ def build_report_model(skill_dir: Path) -> dict: "reviewed_at": output_review_adjudication.get("reviewed_at", ""), "failures": output_review_adjudication.get("failures", []), }, + "benchmark_reproducibility": { + "ok": benchmark_reproducibility.get("ok", False), + "summary": benchmark_reproducibility.get("summary", {}), + "commit": benchmark_reproducibility.get("commit", ""), + "missing_artifacts": benchmark_reproducibility.get("missing_artifacts", []), + "limitations": benchmark_reproducibility.get("limitations", []), + }, "runtime_conformance": conformance.get("summary", {}), "runtime_permissions": { "ok": runtime_permissions.get("ok", False), diff --git a/scripts/yao.py b/scripts/yao.py index c376a27..d80e795 100644 --- a/scripts/yao.py +++ b/scripts/yao.py @@ -465,6 +465,7 @@ def command_report(args: argparse.Namespace) -> int: run_script("render_telemetry_hook_recipes.py", [str(ROOT)]), run_script("render_review_waivers.py", [str(ROOT)]), run_script("render_review_annotations.py", [str(ROOT)]), + run_script("render_benchmark_reproducibility.py", [str(ROOT)]), ] ) report = { @@ -561,6 +562,20 @@ def command_world_class_evidence(args: argparse.Namespace) -> int: return 0 if result["ok"] else 2 +def command_benchmark_reproducibility(args: argparse.Namespace) -> int: + skill_dir = str(Path(args.skill_dir).resolve()) + cmd = [skill_dir] + if args.output_json: + cmd.extend(["--output-json", args.output_json]) + if args.output_md: + cmd.extend(["--output-md", args.output_md]) + if args.generated_at: + cmd.extend(["--generated-at", args.generated_at]) + result = run_script("render_benchmark_reproducibility.py", cmd) + print(json.dumps(result["payload"] if result["payload"] is not None else result, ensure_ascii=False, indent=2)) + return 0 if result["ok"] else 2 + + def command_reference_scan(args: argparse.Namespace) -> int: skill_dir = str(Path(args.skill_dir).resolve()) cmd = [skill_dir] @@ -1181,6 +1196,7 @@ def command_workspace_flow(args: argparse.Namespace) -> int: {"phase": "report-refresh", "result": run_script("render_telemetry_hook_recipes.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_review_waivers.py", [str(ROOT)])}, {"phase": "report-refresh", "result": run_script("render_review_annotations.py", [str(ROOT)])}, + {"phase": "report-refresh", "result": run_script("render_benchmark_reproducibility.py", [str(ROOT)])}, ] ) diff --git a/scripts/yao_cli_parser.py b/scripts/yao_cli_parser.py index ced7328..9a6ab90 100644 --- a/scripts/yao_cli_parser.py +++ b/scripts/yao_cli_parser.py @@ -152,6 +152,16 @@ def build_parser(command_handlers: dict[str, Callable[[argparse.Namespace], int] world_class_evidence_cmd.add_argument("--generated-at") world_class_evidence_cmd.set_defaults(func=_handler(command_handlers, "command_world_class_evidence")) + benchmark_reproducibility_cmd = subparsers.add_parser( + "benchmark-reproducibility", + help="Render benchmark methodology, artifact, failure-disclosure, and reproduction-command evidence.", + ) + benchmark_reproducibility_cmd.add_argument("skill_dir", nargs="?", default=".") + benchmark_reproducibility_cmd.add_argument("--output-json") + benchmark_reproducibility_cmd.add_argument("--output-md") + benchmark_reproducibility_cmd.add_argument("--generated-at") + benchmark_reproducibility_cmd.set_defaults(func=_handler(command_handlers, "command_benchmark_reproducibility")) + reference_scan_cmd = subparsers.add_parser( "reference-scan", help="Render a controlled benchmark scan report for a skill package.", diff --git a/skill-ir/examples/yao-meta-skill.json b/skill-ir/examples/yao-meta-skill.json index eacd5fd..7f4ad6f 100644 --- a/skill-ir/examples/yao-meta-skill.json +++ b/skill-ir/examples/yao-meta-skill.json @@ -136,6 +136,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", diff --git a/skill_atlas/catalog.json b/skill_atlas/catalog.json index 9a746e8..6c1bd05 100644 --- a/skill_atlas/catalog.json +++ b/skill_atlas/catalog.json @@ -48,6 +48,7 @@ "scripts/render_adoption_drift_report.py", "scripts/render_artifact_design_profile.py", "scripts/render_baseline_compare.py", + "scripts/render_benchmark_reproducibility.py", "scripts/render_context_reports.py", "scripts/render_description_drift_history.py", "scripts/render_eval_dashboard.py", diff --git a/tests/verify_benchmark_reproducibility.py b/tests/verify_benchmark_reproducibility.py new file mode 100644 index 0000000..b923085 --- /dev/null +++ b/tests/verify_benchmark_reproducibility.py @@ -0,0 +1,67 @@ +#!/usr/bin/env python3 +import json +import shutil +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent +SCRIPT = ROOT / "scripts" / "render_benchmark_reproducibility.py" +TMP = ROOT / "tests" / "tmp_benchmark_reproducibility" + + +def main() -> None: + shutil.rmtree(TMP, ignore_errors=True) + TMP.mkdir(parents=True, exist_ok=True) + output_json = TMP / "benchmark_reproducibility.json" + output_md = TMP / "benchmark_reproducibility.md" + proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(ROOT), + "--output-json", + str(output_json), + "--output-md", + str(output_md), + "--generated-at", + "2026-06-13", + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + payload = json.loads(proc.stdout) + assert payload["schema_version"] == "1.0", payload + assert payload["ok"] is True, payload + assert payload["summary"]["reproducibility_ready"] is True, payload + assert payload["summary"]["methodology_complete"] is True, payload + assert payload["summary"]["missing_artifact_count"] == 0, payload + assert payload["summary"]["output_case_count"] >= 5, payload + assert payload["summary"]["failure_disclosure_count"] >= 1, payload + assert payload["summary"]["command_count"] >= 10, payload + assert "working_tree_dirty" in payload["summary"], payload + assert payload["git_status"]["available"] is True, payload + assert payload["summary"]["provider_evidence_complete"] is False, payload + assert payload["summary"]["human_review_complete"] is False, payload + assert payload["summary"]["world_class_ready"] is False, payload + headings = {item["heading"]: item["exists"] for item in payload["methodology"]["sections"]} + assert headings["## Benchmark Types"], headings + assert headings["## Failure Disclosure"], headings + artifacts = {item["path"]: item for item in payload["artifacts_checked"]} + assert artifacts["reports/benchmark_methodology.md"]["exists"], artifacts + assert artifacts["evals/failure-cases.md"]["exists"], artifacts + assert artifacts["reports/world_class_evidence_plan.json"]["exists"], artifacts + assert any(command["command"] == "make ci-test" for command in payload["reproduction_commands"]), payload + assert any("provider-backed" in item for item in payload["limitations"]), payload["limitations"] + markdown = output_md.read_text(encoding="utf-8") + assert "Benchmark Reproducibility" in markdown, markdown + assert "reports/benchmark_methodology.md" in markdown, markdown + assert "make ci-test" in markdown, markdown + print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) + + +if __name__ == "__main__": + main() diff --git a/tests/verify_review_studio.py b/tests/verify_review_studio.py index c3898fd..28fc859 100644 --- a/tests/verify_review_studio.py +++ b/tests/verify_review_studio.py @@ -303,6 +303,8 @@ def main() -> None: assert full_payload["evidence_paths"]["output_blind_review"] == "reports/output_blind_review_pack.md", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_review_decisions"] == "reports/output_review_decisions.json", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["output_review_adjudication"] == "reports/output_review_adjudication.md", full_payload["evidence_paths"] + if (ROOT / "reports" / "benchmark_reproducibility.md").exists(): + assert full_payload["evidence_paths"]["benchmark_reproducibility"] == "reports/benchmark_reproducibility.md", full_payload["evidence_paths"] assert full_payload["evidence_paths"]["review_annotations"] == "reports/review_annotations.md", full_payload["evidence_paths"] if (ROOT / "reports" / "world_class_evidence_plan.md").exists(): assert full_payload["evidence_paths"]["world_class_evidence_plan"] == "reports/world_class_evidence_plan.md", full_payload["evidence_paths"] diff --git a/tests/verify_skill_os2_audit.py b/tests/verify_skill_os2_audit.py index b849edf..21dc4ba 100644 --- a/tests/verify_skill_os2_audit.py +++ b/tests/verify_skill_os2_audit.py @@ -39,7 +39,8 @@ def main() -> None: assert payload["summary"]["decision"] == "continue-iteration", payload assert payload["summary"]["world_class_ready"] is False, payload assert payload["summary"]["missing_count"] == 0, payload - assert payload["summary"]["pass_count"] >= 10, payload + assert payload["summary"]["item_count"] >= 15, payload + assert payload["summary"]["pass_count"] >= 11, payload assert payload["summary"]["human_required_count"] >= 1, payload assert payload["summary"]["external_required_count"] >= 3, payload assert payload["artifacts"]["world_class_evidence_plan"] == "reports/world_class_evidence_plan.md", payload @@ -47,16 +48,19 @@ def main() -> None: assert items["skill-ir"]["status"] == "pass", items["skill-ir"] assert items["target-compiler"]["status"] == "pass", items["target-compiler"] assert items["output-eval-lab"]["status"] == "pass", items["output-eval-lab"] + assert items["benchmark-reproducibility"]["status"] == "pass", items["benchmark-reproducibility"] assert items["provider-holdout"]["status"] == "external_required", items["provider-holdout"] assert items["human-adjudication"]["status"] == "human_required", items["human-adjudication"] assert items["native-permission-enforcement"]["status"] == "external_required", items["native-permission-enforcement"] assert items["native-client-telemetry"]["status"] == "external_required", items["native-client-telemetry"] assert any(entry["path"] == "scripts/provider_output_eval_runner.py" and entry["exists"] for entry in items["provider-holdout"]["evidence"]) + assert any(entry["path"] == "reports/benchmark_reproducibility.json" and entry["exists"] for entry in items["benchmark-reproducibility"]["evidence"]) assert any(entry["path"] == "scripts/telemetry_native_host.py" and entry["exists"] for entry in items["native-client-telemetry"]["evidence"]) markdown = output_md.read_text(encoding="utf-8") assert "Skill OS 2.0 Audit" in markdown, markdown assert "reports/world_class_evidence_plan.md" in markdown, markdown assert "`provider-holdout`" in markdown, markdown + assert "Benchmark Reproducibility" in markdown, markdown assert "`human-adjudication`" in markdown, markdown assert "`native-client-telemetry`" in markdown, markdown print(json.dumps({"ok": True}, ensure_ascii=False, indent=2)) diff --git a/tests/verify_skill_overview.py b/tests/verify_skill_overview.py index 1cb3f7e..f4f8eb1 100644 --- a/tests/verify_skill_overview.py +++ b/tests/verify_skill_overview.py @@ -116,6 +116,7 @@ def main() -> None: assert "reports/compiled_targets.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/output_quality_scorecard.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/output_blind_review_pack.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] + assert "reports/benchmark_reproducibility.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/conformance_matrix.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/security_trust_report.md" in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] assert "reports/runtime_permission_probes.md" not in overview_json["skill_summary"]["deliverables"], overview_json["skill_summary"] @@ -137,6 +138,8 @@ def main() -> None: assert overview_json["output_blind_review"]["summary"] == {}, overview_json["output_blind_review"] assert "output_review_adjudication" in overview_json, overview_json.keys() assert overview_json["output_review_adjudication"]["summary"] == {}, overview_json["output_review_adjudication"] + assert "benchmark_reproducibility" in overview_json, overview_json.keys() + assert overview_json["benchmark_reproducibility"]["summary"] == {}, overview_json["benchmark_reproducibility"] assert "runtime_conformance" in overview_json, overview_json.keys() assert "runtime_permissions" in overview_json, overview_json.keys() assert overview_json["runtime_permissions"]["summary"] == {}, overview_json["runtime_permissions"] diff --git a/tests/verify_yao_cli.py b/tests/verify_yao_cli.py index 85c3570..4993342 100644 --- a/tests/verify_yao_cli.py +++ b/tests/verify_yao_cli.py @@ -75,6 +75,7 @@ def main() -> None: assert "review-studio" in parser_help, parser_help assert "skill-os2-audit" in parser_help, parser_help assert "world-class-evidence" in parser_help, parser_help + assert "benchmark-reproducibility" in parser_help, parser_help assert "telemetry-import" in parser_help, parser_help assert "telemetry-emit" in parser_help, parser_help assert "telemetry-hooks" in parser_help, parser_help @@ -179,6 +180,20 @@ def main() -> None: assert world_class_evidence_result["payload"]["summary"]["decision"] == "collect-external-evidence", world_class_evidence_result assert world_class_evidence_result["payload"]["summary"]["ready_to_claim_world_class"] is False, world_class_evidence_result + benchmark_reproducibility_result = run( + "benchmark-reproducibility", + str(ROOT), + "--output-json", + str(tmp_root / "benchmark_reproducibility.json"), + "--output-md", + str(tmp_root / "benchmark_reproducibility.md"), + "--generated-at", + "2026-06-13", + ) + assert benchmark_reproducibility_result["ok"], benchmark_reproducibility_result + assert benchmark_reproducibility_result["payload"]["summary"]["reproducibility_ready"] is True, benchmark_reproducibility_result + assert benchmark_reproducibility_result["payload"]["summary"]["world_class_ready"] is False, benchmark_reproducibility_result + quickstart_result = run( "quickstart", "--output-dir",
路径Path作用Role类型Type
SKILL.mdSkill 入口文件Skill entrypoint文件file
README.md人类可读使用说明Human-readable usage guide文件file
agents/interface.yaml跨平台接口元数据Neutral interface metadata文件file
manifest.json生命周期与打包元数据Lifecycle and portability metadata文件file
references扩展指导与复用资料Extended guidance and reusable notes目录folder
scripts确定性脚本或本地工具Deterministic helpers or local tooling目录folder
evals触发与质量检查Trigger and quality checks目录folder
reports生成的证据与总结报告Generated evidence and overview artifacts目录folder