frostbyte_neo/yao-meta-skill
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity

Separate source and generated release locks

Browse Source
This commit is contained in:
yaojingang
2026-06-17 04:23:04 +08:00
parent 4a1a529570
commit ed6960f65d
45 changed files with 794 additions and 483 deletions
Download Patch File Download Diff File Expand all files Collapse all files
scripts/benchmark_release_lock.py
+98
View File
@@ -0,0 +1,98 @@
#!/usr/bin/env python3
"""Git source/generated dirty classification for benchmark release locks."""
import subprocess
from pathlib import Path
from typing import Any
SCRIPT_INTERFACE = "internal-module"
SCRIPT_INTERFACE_REASON = "Imported by render_benchmark_reproducibility.py to keep release-lock git status classification out of the report renderer."
GENERATED_DIRTY_PREFIXES = (
"dist/",
"registry/index.json",
"registry/packages/",
"reports/",
"skill_atlas/",
"skill-ir/examples/",
)
def status_path(line: str) -> str:
path = line[3:].strip() if len(line) > 3 else line.strip()
if " -> " in path:
path = path.rsplit(" -> ", 1)[-1].strip()
return path
def is_generated_dirty_path(path: str) -> bool:
return any(path == prefix.rstrip("/") or path.startswith(prefix) for prefix in GENERATED_DIRTY_PREFIXES)
def git_status(skill_dir: Path) -> dict[str, Any]:
try:
proc = subprocess.run(
["git", "status", "--short", "--untracked-files=all"],
cwd=skill_dir,
capture_output=True,
text=True,
check=True,
)
except (OSError, subprocess.CalledProcessError):
return {
"available": False,
"dirty": None,
"changed_file_count": None,
"source_dirty": None,
"source_changed_file_count": None,
"generated_dirty": None,
"generated_changed_file_count": None,
}
lines = [line for line in proc.stdout.splitlines() if line.strip()]
paths = [status_path(line) for line in lines]
generated_lines = [line for line, path in zip(lines, paths) if is_generated_dirty_path(path)]
source_lines = [line for line, path in zip(lines, paths) if not is_generated_dirty_path(path)]
return {
"available": True,
"dirty": bool(lines),
"changed_file_count": len(lines),
"generated_dirty": bool(generated_lines),
"generated_changed_file_count": len(generated_lines),
"source_dirty": bool(source_lines),
"source_changed_file_count": len(source_lines),
"sample": lines[:12],
"source_sample": source_lines[:12],
"generated_sample": generated_lines[:12],
"generated_dirty_prefixes": list(GENERATED_DIRTY_PREFIXES),
"scope": "generation-time status before this report is written",
}
def release_lock_status(status: dict[str, Any], commit: str) -> dict[str, Any]:
available = status.get("available") is True
source_clean = status.get("source_dirty") is False
known_commit = bool(commit and commit != "unknown")
ready = available and source_clean and known_commit
reasons = []
if not available:
reasons.append("git status unavailable")
if not known_commit:
reasons.append("git commit unavailable")
if status.get("source_dirty") is True:
reasons.append("source files were dirty at generation time")
if status.get("source_dirty") is None:
reasons.append("working tree cleanliness unknown")
if status.get("generated_dirty") is True and status.get("source_dirty") is False:
reasons.append("only generated evidence artifacts were dirty at generation time")
if not reasons:
reasons.append("clean source tree at generation-time HEAD")
return {
"ready": ready,
"commit": commit,
"status_scope": status.get("scope", "generation-time status"),
"source_changed_file_count": status.get("source_changed_file_count"),
"generated_changed_file_count": status.get("generated_changed_file_count"),
"reason": "; ".join(reasons),
}
scripts/render_benchmark_reproducibility.py
+11 -47
View File
@@ -7,6 +7,7 @@ from datetime import date
from pathlib import Path
from typing import Any
from benchmark_release_lock import git_status, release_lock_status
ROOT = Path(__file__).resolve().parent.parent
SCRIPT_INTERFACE = "cli"
@@ -207,27 +208,6 @@ def git_commit(skill_dir: Path) -> str:
return proc.stdout.strip() or "unknown"
def git_status(skill_dir: Path) -> dict[str, Any]:
try:
proc = subprocess.run(
["git", "status", "--short", "--untracked-files=all"],
cwd=skill_dir,
capture_output=True,
text=True,
check=True,
)
except (OSError, subprocess.CalledProcessError):
return {"available": False, "dirty": None, "changed_file_count": None}
lines = [line for line in proc.stdout.splitlines() if line.strip()]
return {
"available": True,
"dirty": bool(lines),
"changed_file_count": len(lines),
"sample": lines[:12],
"scope": "generation-time status before this report is written",
}
def count_jsonl(path: Path) -> int:
if not path.exists():
return 0
@@ -292,30 +272,6 @@ def evidence_bundle_fingerprint(artifacts: list[dict[str, Any]]) -> dict[str, An
}
def release_lock_status(status: dict[str, Any], commit: str) -> dict[str, Any]:
available = status.get("available") is True
clean = status.get("dirty") is False
known_commit = bool(commit and commit != "unknown")
ready = available and clean and known_commit
reasons = []
if not available:
reasons.append("git status unavailable")
if not known_commit:
reasons.append("git commit unavailable")
if status.get("dirty") is True:
reasons.append("working tree was dirty at generation time")
if status.get("dirty") is None:
reasons.append("working tree cleanliness unknown")
if not reasons:
reasons.append("clean generation-time HEAD")
return {
"ready": ready,
"commit": commit,
"status_scope": status.get("scope", "generation-time status"),
"reason": "; ".join(reasons),
}
def public_claim_blockers(
local_reproducibility_ready: bool,
release_lock_ready: bool,
@@ -444,6 +400,10 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
"public_claim_blocker_count": len(claim_blockers),
"working_tree_dirty": status.get("dirty"),
"changed_file_count": status.get("changed_file_count"),
"source_tree_dirty": status.get("source_dirty"),
"source_changed_file_count": status.get("source_changed_file_count"),
"generated_tree_dirty": status.get("generated_dirty"),
"generated_changed_file_count": status.get("generated_changed_file_count"),
},
"public_claim": {
"ready": public_claim_ready,
@@ -463,7 +423,7 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
"policy": "Keep representative failures visible and tied to regression checks.",
},
"limitations": [
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
"The git commit and dirty flags are generation-time context; release lock is blocked by source changes, while generated evidence artifacts are tracked separately.",
"Local command-runner evidence is reproducible but does not replace provider-backed model holdout evidence.",
"Pending blind-review decisions are visible but do not count as human adjudication.",
"World-class readiness remains false until external and human evidence gaps close.",
@@ -483,6 +443,8 @@ def render_markdown(report: dict[str, Any]) -> str:
f"Generated at: `{report['generated_at']}`",
f"Commit: `{report['commit']}`",
f"Working tree dirty at generation: `{str(summary.get('working_tree_dirty')).lower()}`",
f"Source tree dirty at generation: `{str(summary.get('source_tree_dirty')).lower()}`",
f"Generated evidence dirty at generation: `{str(summary.get('generated_tree_dirty')).lower()}`",
f"Evidence bundle SHA256: `{summary.get('evidence_bundle_sha256', '')}`",
"",
"## Summary",
@@ -504,8 +466,10 @@ def render_markdown(report: dict[str, Any]) -> str:
f"- public claim ready: `{str(summary['public_claim_ready']).lower()}`",
f"- public claim blockers: `{summary['public_claim_blocker_count']}`",
f"- changed files at generation: `{summary.get('changed_file_count')}`",
f"- source changed files at generation: `{summary.get('source_changed_file_count')}`",
f"- generated changed files at generation: `{summary.get('generated_changed_file_count')}`",
"",
"This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.",
"This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit and dirty samples are generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.",
"",
"## Public Claim Boundary",
"",
scripts/render_evidence_consistency.py
+6 -3
View File
@@ -156,11 +156,14 @@ def build_report(skill_dir: Path, generated_at: str) -> dict[str, Any]:
compare_values(
checks,
key="benchmark-release-lock-self-consistency",
label="Benchmark release lock matches git dirty state",
expected=not bool(nested(benchmark, ["git_status", "dirty"], True)),
label="Benchmark release lock matches source dirty state",
expected=not bool(nested(benchmark, ["git_status", "source_dirty"], True)),
actual=benchmark_summary.get("release_lock_ready"),
paths=[REQUIRED_REPORTS["benchmark"]],
detail="The benchmark release lock must reflect the generation-time git dirty flag.",
detail=(
"The benchmark release lock must be blocked by source changes, while generated evidence artifacts are "
"tracked as generation context."
),
)
current_git_status = git_worktree_status(skill_dir)
if current_git_status.get("available") and current_git_status.get("clean") is True: