frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Harden release evidence flow checks

Browse Source
This commit is contained in:
yaojingang
2026-06-16 01:56:03 +08:00
parent 4187d92ac3
commit 959fcaee08
16 changed files with 81 additions and 45 deletions
Download Patch File Download Diff File Expand all files Collapse all files
AGENTS.md
+1
View File
@@ -48,6 +48,7 @@ python3 scripts/render_world_class_submission_review.py . --generated-at "$GENER
python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT"
python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT"
python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT"
python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"
python3 scripts/render_skill_overview.py .
python3 scripts/render_skill_interpretation.py .
python3 scripts/render_review_viewer.py .
registry/index.json
+1 -1
View File
@@ -16,7 +16,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
}
]
}
registry/packages/yao-meta-skill.json
+3 -3
View File
@@ -16,8 +16,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d",
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d"
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee",
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050"
},
"compatibility": {
"openai": "pass",
@@ -48,7 +48,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d",
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
reports/output_execution_runs.json
+10 -10
View File
@@ -34,7 +34,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.29,
"duration_ms": 27.11,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -62,7 +62,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.71,
"duration_ms": 26.32,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -85,7 +85,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.61,
"duration_ms": 25.75,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -113,7 +113,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.54,
"duration_ms": 28.01,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -136,7 +136,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 30.87,
"duration_ms": 29.77,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -164,7 +164,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.9,
"duration_ms": 28.46,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -187,7 +187,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.08,
"duration_ms": 26.53,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -214,7 +214,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.5,
"duration_ms": 26.46,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -237,7 +237,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 31.33,
"duration_ms": 25.86,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
@@ -266,7 +266,7 @@
"execution_mode": "command",
"model_executed": false,
"command_executed": true,
"duration_ms": 29.65,
"duration_ms": 25.67,
"provider": "local-output-eval-runner",
"model": "",
"usage": {
reports/output_execution_runs.md
+10 -10
View File
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
| skill-package-contract | baseline | command | local-output-eval-runner | 30.29 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 30.71 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 31.61 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 31.54 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 30.87 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 29.9 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 31.08 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 31.5 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 31.33 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 29.65 | 98 | 100.0 | pass |
| skill-package-contract | baseline | command | local-output-eval-runner | 27.11 | 33 | 0.0 | pass |
| skill-package-contract | with_skill | command | local-output-eval-runner | 26.32 | 73 | 100.0 | pass |
| output-eval-expectation | baseline | command | local-output-eval-runner | 25.75 | 36 | 0.0 | pass |
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.01 | 80 | 100.0 | pass |
| ir-before-packaging | baseline | command | local-output-eval-runner | 29.77 | 33 | 0.0 | pass |
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.46 | 80 | 100.0 | pass |
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 26.53 | 36 | 0.0 | pass |
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 26.46 | 65 | 100.0 | pass |
| file-backed-governed-package | baseline | command | local-output-eval-runner | 25.86 | 37 | 0.0 | pass |
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 25.67 | 98 | 100.0 | pass |
## Next Fixes
reports/package_verification.json
+1 -1
View File
@@ -8,7 +8,7 @@
"target_count": 4,
"adapter_count": 4,
"archive_present": true,
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d",
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050",
"archive_entry_count": 623,
"failure_count": 0,
"warning_count": 0
reports/package_verification.md
+1 -1
View File
@@ -4,7 +4,7 @@
- Package directory: `dist`
- Targets: `4 / 4` adapters present
- Archive present: `True`
- Archive SHA256: `729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d`
- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050`
- Failures: `0`
- Warnings: `0`
reports/registry_audit.json
+4 -4
View File
@@ -21,8 +21,8 @@
"trust_level": "local",
"license": "MIT",
"checksums": {
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d",
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d"
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee",
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050"
},
"compatibility": {
"openai": "pass",
@@ -53,7 +53,7 @@
},
"distribution": {
"archive_verified": true,
"archive_sha256": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d",
"archive_sha256": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050",
"package_verification": "reports/package_verification.json",
"install_simulated": true,
"install_simulation": "reports/install_simulation.json"
@@ -78,7 +78,7 @@
"vscode"
],
"package_metadata": "registry/packages/yao-meta-skill.json",
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
}
]
},
reports/registry_audit.md
+2 -2
View File
@@ -6,8 +6,8 @@
- Maturity: `governed`
- Owner: `Yao Team`
- License: `MIT`
- Package SHA256: `c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d`
- Archive SHA256: `729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d`
- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee`
- Archive SHA256: `19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050`
- Install simulated: `True`
## Compatibility
reports/security_trust_report.json
+1 -1
View File
@@ -23,7 +23,7 @@
"interactive_script_count": 0,
"package_hash_scope": "source-contract-without-generated-reports",
"package_hash_file_count": 200,
"package_sha256": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
"package_sha256": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
},
"failures": [],
"warnings": [],
reports/security_trust_report.md
+1 -1
View File
@@ -16,7 +16,7 @@
- Interactive scripts: `0`
- Package hash scope: `source-contract-without-generated-reports`
- Package hash files: `200`
- Package SHA256: `c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d`
- Package SHA256: `fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee`
## Failures
reports/skill_os2_coverage.json
+1 -1
View File
@@ -313,7 +313,7 @@
"label": "Benchmark Methodology",
"status": "pass",
"objective": "Recommended Skill OS 2.0 implementation PR from the upgrade plan.",
"current": "24 required artifacts checked",
"current": "25 required artifacts checked",
"command": "make ci-test",
"test": "tests/verify_benchmark_reproducibility.py",
"evidence": [
reports/skill_os2_coverage.md
+1 -1
View File
@@ -36,7 +36,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts,
| Item | Status | Current | Command | Test |
| --- | --- | --- | --- | --- |
| Benchmark Methodology | `pass` | 24 required artifacts checked | `make ci-test` | `tests/verify_benchmark_reproducibility.py` |
| Benchmark Methodology | `pass` | 25 required artifacts checked | `make ci-test` | `tests/verify_benchmark_reproducibility.py` |
| Output Eval Schema | `pass` | 5 output cases | `make ci-test` | `tests/verify_output_eval_lab.py` |
| Output Eval Runner | `pass` | delta 100.0 | `make ci-test` | `tests/verify_output_eval_lab.py` |
| Output Quality Scorecard | `pass` | gate pass True | `make ci-test` | `tests/verify_output_eval_lab.py` |
reports/upgrade_check.json
+2 -2
View File
@@ -70,12 +70,12 @@
{
"field": "archive_sha256",
"from": "",
"to": "729d4a307944e544a830974363f436149fc647f9eb12e6e712d66a96f79b022d"
"to": "19810129f02c71eb00d04a5eb9c9570febded99beac0230de82a09610e027050"
},
{
"field": "package_sha256",
"from": "0000000000000000000000000000000000000000000000000000000000000000",
"to": "c031eba962def76fe31e98da4220d9db7f58a1ed8c6c6d1c8906d71f0c24c93d"
"to": "fa285e57f36fb8f33b0c9cdc5de924532a4ec3f30be5880fd3bcde6d53d363ee"
}
]
},
scripts/evidence_consistency_release.py
+9 -7
View File
@@ -9,7 +9,8 @@ SOURCE_REFRESH_HEADER = "After source changes that affect scripts"
CLEAN_LOCK_HEADER = "For final release evidence"
CLEAN_LOCK_END = "If `reports/benchmark_reproducibility.json`"
FIRST_CLASS_REPORT_COMMANDS = [
SOURCE_REFRESH_REPORT_COMMANDS = [
'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"',
"python3 scripts/render_skill_overview.py .",
"python3 scripts/render_skill_interpretation.py .",
"python3 scripts/render_review_viewer.py .",
@@ -17,6 +18,7 @@ FIRST_CLASS_REPORT_COMMANDS = [
"python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json",
'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"',
]
CLEAN_LOCK_REPORT_COMMANDS = list(SOURCE_REFRESH_REPORT_COMMANDS)
def section_between(text: str, start: str, end: str) -> str:
@@ -28,8 +30,8 @@ def section_between(text: str, start: str, end: str) -> str:
return section
def command_presence(section: str) -> dict[str, bool]:
return {command: command in section for command in FIRST_CLASS_REPORT_COMMANDS}
def command_presence(section: str, commands: list[str]) -> dict[str, bool]:
return {command: command in section for command in commands}
def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]:
@@ -41,15 +43,15 @@ def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]:
"AGENTS.md": True,
"source_refresh_section": True,
"clean_lock_section": True,
"source_refresh_commands": {command: True for command in FIRST_CLASS_REPORT_COMMANDS},
"clean_lock_commands": {command: True for command in FIRST_CLASS_REPORT_COMMANDS},
"source_refresh_commands": {command: True for command in SOURCE_REFRESH_REPORT_COMMANDS},
"clean_lock_commands": {command: True for command in CLEAN_LOCK_REPORT_COMMANDS},
}
actual = {
"AGENTS.md": agents_path.exists(),
"source_refresh_section": bool(source_refresh),
"clean_lock_section": bool(clean_lock),
"source_refresh_commands": command_presence(source_refresh),
"clean_lock_commands": command_presence(clean_lock),
"source_refresh_commands": command_presence(source_refresh, SOURCE_REFRESH_REPORT_COMMANDS),
"clean_lock_commands": command_presence(clean_lock, CLEAN_LOCK_REPORT_COMMANDS),
}
return {
"key": "release-evidence-flow-covers-first-class-reports",
tests/verify_evidence_consistency.py
+33
View File
@@ -102,12 +102,14 @@ def assert_release_evidence_instructions_cover_first_class_reports() -> None:
agents_text = (ROOT / "AGENTS.md").read_text(encoding="utf-8")
source_refresh_header = "After source changes that affect scripts"
clean_lock_header = "For final release evidence"
benchmark_command = 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"'
assert source_refresh_header in agents_text, agents_text
assert clean_lock_header in agents_text, agents_text
source_refresh = agents_text.split(source_refresh_header, 1)[1].split(clean_lock_header, 1)[0]
clean_lock = agents_text.split(clean_lock_header, 1)[1].split("If `reports/benchmark_reproducibility.json`", 1)[0]
for block in [source_refresh, clean_lock]:
assert benchmark_command in block, block
assert "python3 scripts/render_skill_interpretation.py ." in block, block
assert "python3 scripts/render_world_class_preflight.py . --generated-at \"$GENERATED_AT\"" in block, block
assert "python3 scripts/render_evidence_consistency.py . --generated-at \"$GENERATED_AT\"" in block, block
@@ -274,6 +276,37 @@ def main() -> None:
release_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]
)
benchmark_flow_drift_root = TMP / "benchmark-flow-drift-skill"
copy_reports(benchmark_flow_drift_root)
agents_path = benchmark_flow_drift_root / "AGENTS.md"
agents_text = agents_path.read_text(encoding="utf-8")
clean_lock_header = "For final release evidence"
benchmark_command = 'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"\n'
prefix, suffix = agents_text.split(clean_lock_header, 1)
agents_path.write_text(
prefix + clean_lock_header + suffix.replace(benchmark_command, "", 1),
encoding="utf-8",
)
benchmark_flow_drift_proc = run(
[
sys.executable,
str(SCRIPT),
str(benchmark_flow_drift_root),
"--output-json",
str(TMP / "benchmark_flow_drift.json"),
"--output-md",
str(TMP / "benchmark_flow_drift.md"),
"--generated-at",
"2026-06-15",
]
)
assert benchmark_flow_drift_proc.returncode == 2, benchmark_flow_drift_proc.stdout
benchmark_flow_drift_payload = json.loads(benchmark_flow_drift_proc.stdout)
benchmark_flow_drift_checks = {item["key"]: item for item in benchmark_flow_drift_payload["checks"]}
assert benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]["status"] == "fail", (
benchmark_flow_drift_checks["release-evidence-flow-covers-first-class-reports"]
)
workflow_drift_root = TMP / "workflow-drift-skill"
copy_reports(workflow_drift_root)
studio_path = workflow_drift_root / "reports" / "review-studio.json"