diff --git a/AGENTS.md b/AGENTS.md index 351b961..223cb81 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -48,6 +48,7 @@ python3 scripts/render_world_class_submission_review.py . --generated-at "$GENER python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT" python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT" +python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT" python3 scripts/render_skill_overview.py . python3 scripts/render_skill_interpretation.py . python3 scripts/render_review_viewer.py . diff --git a/registry/index.json b/registry/index.json index 57b8348..a8b4da6 100644 --- a/registry/index.json +++ b/registry/index.json @@ -16,7 +16,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d" + "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" } ] } diff --git a/registry/packages/yao-meta-skill.json b/registry/packages/yao-meta-skill.json index d4ed9d2..22a9d80 100644 --- a/registry/packages/yao-meta-skill.json +++ b/registry/packages/yao-meta-skill.json @@ -16,8 +16,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d", - "archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d" + "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee", + "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050" }, "compatibility": { "openai": "pass", @@ -48,7 +48,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d", + "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index a909801..51e94aa 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.29, + "duration_ms": 27.11, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.71, + "duration_ms": 26.32, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.61, + "duration_ms": 25.75, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.54, + "duration_ms": 28.01, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 30.87, + "duration_ms": 29.77, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.9, + "duration_ms": 28.46, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.08, + "duration_ms": 26.53, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.5, + "duration_ms": 26.46, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 31.33, + "duration_ms": 25.86, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 29.65, + "duration_ms": 25.67, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 9762bc8..74397ba 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 30.29 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 30.71 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 31.61 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.54 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 30.87 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 29.9 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 31.08 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 31.5 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 31.33 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 29.65 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 27.11 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 26.32 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 25.75 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.01 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 29.77 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.46 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.53 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.46 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.86 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.67 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/package_verification.json b/reports/package_verification.json index 4d40246..2263dd5 100644 --- a/reports/package_verification.json +++ b/reports/package_verification.json @@ -8,7 +8,7 @@ "target_count": 4, "adapter_count": 4, "archive_present": true, - "archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d", + "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050", "archive_entry_count": 623, "failure_count": 0, "warning_count": 0 diff --git a/reports/package_verification.md b/reports/package_verification.md index 87f8d7a..afc59f2 100644 --- a/reports/package_verification.md +++ b/reports/package_verification.md @@ -4,7 +4,7 @@ - Package directory: `dist` - Targets: `4 / 4` adapters present - Archive present: `True` -- Archive SHA256: `729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d` +- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050` - Failures: `0` - Warnings: `0` diff --git a/reports/registry_audit.json b/reports/registry_audit.json index 97b159b..7524321 100644 --- a/reports/registry_audit.json +++ b/reports/registry_audit.json @@ -21,8 +21,8 @@ "trust_level": "local", "license": "MIT", "checksums": { - "package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d", - "archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d" + "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee", + "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050" }, "compatibility": { "openai": "pass", @@ -53,7 +53,7 @@ }, "distribution": { "archive_verified": true, - "archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d", + "archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050", "package_verification": "reports/package_verification.json", "install_simulated": true, "install_simulation": "reports/install_simulation.json" @@ -78,7 +78,7 @@ "vscode" ], "package_metadata": "registry/packages/yao-meta-skill.json", - "package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d" + "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" } ] }, diff --git a/reports/registry_audit.md b/reports/registry_audit.md index 7193fbb..6064891 100644 --- a/reports/registry_audit.md +++ b/reports/registry_audit.md @@ -6,8 +6,8 @@ - Maturity: `governed` - Owner: `Yao Team` - License: `MIT` -- Package SHA256: `c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d` -- Archive SHA256: `729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d` +- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee` +- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050` - Install simulated: `True` ## Compatibility diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 4578364..64c158e 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -23,7 +23,7 @@ "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", "package_hash_file_count": 200, - "package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d" + "package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" }, "failures": [], "warnings": [], diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index bc4ab3e..19173af 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -16,7 +16,7 @@ - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` - Package hash files: `200` -- Package SHA256: `c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d` +- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee` ## Failures diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index 3f3ea13..c373c3b 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -313,7 +313,7 @@ "label": "Benchmark Methodology", "status": "pass", "objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.", - "current": "24 required artifacts checked", + "current": "25 required artifacts checked", "command": "make ci-test", "test": "tests/verify_benchmark_reproducibility.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index bc979cf..b4341b1 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -36,7 +36,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Item | Status | Current | Command | Test | | --- | --- | --- | --- | --- | -| Benchmark Methodology | `pass` | 24 required artifacts checked | `make ci-test` | `tests/verify_benchmark_reproducibility.py` | +| Benchmark Methodology | `pass` | 25 required artifacts checked | `make ci-test` | `tests/verify_benchmark_reproducibility.py` | | Output Eval Schema | `pass` | 5 output cases | `make ci-test` | `tests/verify_output_eval_lab.py` | | Output Eval Runner | `pass` | delta 100.0 | `make ci-test` | `tests/verify_output_eval_lab.py` | | Output Quality Scorecard | `pass` | gate pass True | `make ci-test` | `tests/verify_output_eval_lab.py` | diff --git a/reports/upgrade_check.json b/reports/upgrade_check.json index afa4b02..8066c29 100644 --- a/reports/upgrade_check.json +++ b/reports/upgrade_check.json @@ -70,12 +70,12 @@ { "field": "archive_sha256", "from": "", - "to": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d" + "to": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050" }, { "field": "package_sha256", "from": "0000000000000000000000000000000000000000000000000000000000000000", - "to": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d" + "to": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee" } ] }, diff --git a/scripts/evidence_consistency_release.py b/scripts/evidence_consistency_release.py index 7a8145d..a73186a 100644 --- a/scripts/evidence_consistency_release.py +++ b/scripts/evidence_consistency_release.py @@ -9,7 +9,8 @@ SOURCE_REFRESH_HEADER = "After source changes that affect scripts" CLEAN_LOCK_HEADER = "For final release evidence" CLEAN_LOCK_END = "If `reports/benchmark_reproducibility.json`" -FIRST_CLASS_REPORT_COMMANDS = [ +SOURCE_REFRESH_REPORT_COMMANDS = [ + 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"', "python3 scripts/render_skill_overview.py .", "python3 scripts/render_skill_interpretation.py .", "python3 scripts/render_review_viewer.py .", @@ -17,6 +18,7 @@ FIRST_CLASS_REPORT_COMMANDS = [ "python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json", 'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"', ] +CLEAN_LOCK_REPORT_COMMANDS = list(SOURCE_REFRESH_REPORT_COMMANDS) def section_between(text: str, start: str, end: str) -> str: @@ -28,8 +30,8 @@ def section_between(text: str, start: str, end: str) -> str: return section -def command_presence(section: str) -> dict[str, bool]: - return {command: command in section for command in FIRST_CLASS_REPORT_COMMANDS} +def command_presence(section: str, commands: list[str]) -> dict[str, bool]: + return {command: command in section for command in commands} def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]: @@ -41,15 +43,15 @@ def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]: "AGENTS.md": True, "source_refresh_section": True, "clean_lock_section": True, - "source_refresh_commands": {command: True for command in FIRST_CLASS_REPORT_COMMANDS}, - "clean_lock_commands": {command: True for command in FIRST_CLASS_REPORT_COMMANDS}, + "source_refresh_commands": {command: True for command in SOURCE_REFRESH_REPORT_COMMANDS}, + "clean_lock_commands": {command: True for command in CLEAN_LOCK_REPORT_COMMANDS}, } actual = { "AGENTS.md": agents_path.exists(), "source_refresh_section": bool(source_refresh), "clean_lock_section": bool(clean_lock), - "source_refresh_commands": command_presence(source_refresh), - "clean_lock_commands": command_presence(clean_lock), + "source_refresh_commands": command_presence(source_refresh, SOURCE_REFRESH_REPORT_COMMANDS), + "clean_lock_commands": command_presence(clean_lock, CLEAN_LOCK_REPORT_COMMANDS), } return { "key": "release-evidence-flow-covers-first-class-reports", diff --git a/tests/verify_evidence_consistency.py b/tests/verify_evidence_consistency.py index 7c45da7..e68ef09 100644 --- a/tests/verify_evidence_consistency.py +++ b/tests/verify_evidence_consistency.py @@ -102,12 +102,14 @@ def assert_release_evidence_instructions_cover_first_class_reports() -> None: agents_text = (ROOT / "AGENTS.md").read_text(encoding="utf-8") source_refresh_header = "After source changes that affect scripts" clean_lock_header = "For final release evidence" + benchmark_command = 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"' assert source_refresh_header in agents_text, agents_text assert clean_lock_header in agents_text, agents_text source_refresh = agents_text.split(source_refresh_header, 1)[1].split(clean_lock_header, 1)[0] clean_lock = agents_text.split(clean_lock_header, 1)[1].split("If `reports/benchmark_reproducibility.json`", 1)[0] for block in [source_refresh, clean_lock]: + assert benchmark_command in block, block assert "python3 scripts/render_skill_interpretation.py ." in block, block assert "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"" in block, block assert "python3 scripts/render_evidence_consistency.py . --generated-at \"$GENERATED_AT\"" in block, block @@ -274,6 +276,37 @@ def main() -> None: release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"] ) + benchmark_flow_drift_root = TMP / "benchmark-flow-drift-skill" + copy_reports(benchmark_flow_drift_root) + agents_path = benchmark_flow_drift_root / "AGENTS.md" + agents_text = agents_path.read_text(encoding="utf-8") + clean_lock_header = "For final release evidence" + benchmark_command = 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"\n' + prefix, suffix = agents_text.split(clean_lock_header, 1) + agents_path.write_text( + prefix + clean_lock_header + suffix.replace(benchmark_command, "", 1), + encoding="utf-8", + ) + benchmark_flow_drift_proc = run( + [ + sys.executable, + str(SCRIPT), + str(benchmark_flow_drift_root), + "--output-json", + str(TMP / "benchmark_flow_drift.json"), + "--output-md", + str(TMP / "benchmark_flow_drift.md"), + "--generated-at", + "2026-06-15", + ] + ) + assert benchmark_flow_drift_proc.returncode == 2, benchmark_flow_drift_proc.stdout + benchmark_flow_drift_payload = json.loads(benchmark_flow_drift_proc.stdout) + benchmark_flow_drift_checks = {item["key"]: item for item in benchmark_flow_drift_payload["checks"]} + assert benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]["status"] == "fail", ( + benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"] + ) + workflow_drift_root = TMP / "workflow-drift-skill" copy_reports(workflow_drift_root) studio_path = workflow_drift_root / "reports" / "review-studio.json"