yaojingang
127 lines
7.4 KiB
Python
127 lines
7.4 KiB
Python
from pathlib import Path
|
|
from typing import Any
|
|
|
|
|
|
SCRIPT_INTERFACE = "internal-module"
|
|
SCRIPT_INTERFACE_REASON = "Imported by render_evidence_consistency.py to verify release evidence refresh instructions."
|
|
|
|
SOURCE_REFRESH_HEADER = "After source changes that affect scripts"
|
|
CLEAN_LOCK_HEADER = "For final release evidence"
|
|
CLEAN_LOCK_END = "If `reports/benchmark_reproducibility.json`"
|
|
|
|
SOURCE_REFRESH_REPORT_COMMANDS = [
|
|
'python3 scripts/run_output_execution.py --runner-command \'["python3","scripts/local_output_eval_runner.py"]\'',
|
|
'python3 scripts/compile_skill.py . --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --platform vscode --expectations evals/packaging_expectations.json --output-dir dist --zip",
|
|
'python3 scripts/simulate_install.py . --package-dir dist --install-root dist/install-simulation --output-json reports/install_simulation.json --output-md reports/install_simulation.md --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/trust_check.py . --output-json reports/security_trust_report.json --output-md reports/security_trust_report.md",
|
|
'python3 scripts/registry_audit.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/verify_package.py . --package-dir dist --expectations evals/packaging_expectations.json --registry-json reports/registry_audit.json --output-json reports/package_verification.json --output-md reports/package_verification.md --require-zip --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/upgrade_check.py . --previous-package-json registry/examples/yao-meta-skill-1.0.0.json --current-package-json reports/registry_audit.json --output-json reports/upgrade_check.json --output-md reports/upgrade_check.md --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_adoption_drift_report.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_architecture_maintainability.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/python_compat_check.py . --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/probe_runtime_permissions.py . --package-dir dist",
|
|
'python3 scripts/render_review_waivers.py . --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/render_review_annotations.py .",
|
|
'python3 scripts/build_skill_atlas.py --workspace-root . --output-dir skill_atlas --report-html reports/skill_atlas.html --report-json reports/skill_atlas.json --today "$GENERATED_AT"',
|
|
'python3 scripts/render_world_class_evidence_plan.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_world_class_evidence_ledger.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_world_class_evidence_intake.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_world_class_submission_review.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_world_class_operator_runbook.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_world_class_claim_guard.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_skill_os2_audit.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_skill_os2_coverage.py . --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/render_skill_overview.py .",
|
|
"python3 scripts/render_skill_interpretation.py .",
|
|
"python3 scripts/render_review_viewer.py .",
|
|
'python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json",
|
|
'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"',
|
|
]
|
|
CLEAN_LOCK_REPORT_COMMANDS = [
|
|
'python3 scripts/render_context_reports.py --generated-at "$GENERATED_AT"',
|
|
'python3 scripts/render_benchmark_reproducibility.py . --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/render_skill_overview.py .",
|
|
"python3 scripts/render_skill_interpretation.py .",
|
|
"python3 scripts/render_review_viewer.py .",
|
|
'python3 scripts/render_world_class_preflight.py . --generated-at "$GENERATED_AT"',
|
|
"python3 scripts/render_review_studio.py . --output-html reports/review-studio.html --output-json reports/review-studio.json",
|
|
'python3 scripts/render_evidence_consistency.py . --generated-at "$GENERATED_AT"',
|
|
]
|
|
|
|
|
|
def section_between(text: str, start: str, end: str) -> str:
|
|
if start not in text:
|
|
return ""
|
|
section = text.split(start, 1)[1]
|
|
if end in section:
|
|
section = section.split(end, 1)[0]
|
|
return section
|
|
|
|
|
|
def command_presence(section: str, commands: list[str]) -> dict[str, bool]:
|
|
return {command: command in section for command in commands}
|
|
|
|
|
|
def build_release_evidence_flow_check(skill_dir: Path) -> dict[str, Any]:
|
|
agents_path = skill_dir / "AGENTS.md"
|
|
agents_text = agents_path.read_text(encoding="utf-8") if agents_path.exists() else ""
|
|
source_refresh = section_between(agents_text, SOURCE_REFRESH_HEADER, CLEAN_LOCK_HEADER)
|
|
clean_lock = section_between(agents_text, CLEAN_LOCK_HEADER, CLEAN_LOCK_END)
|
|
expected = {
|
|
"AGENTS.md": True,
|
|
"source_refresh_section": True,
|
|
"clean_lock_section": True,
|
|
"source_refresh_commands": {command: True for command in SOURCE_REFRESH_REPORT_COMMANDS},
|
|
"clean_lock_commands": {command: True for command in CLEAN_LOCK_REPORT_COMMANDS},
|
|
}
|
|
actual = {
|
|
"AGENTS.md": agents_path.exists(),
|
|
"source_refresh_section": bool(source_refresh),
|
|
"clean_lock_section": bool(clean_lock),
|
|
"source_refresh_commands": command_presence(source_refresh, SOURCE_REFRESH_REPORT_COMMANDS),
|
|
"clean_lock_commands": command_presence(clean_lock, CLEAN_LOCK_REPORT_COMMANDS),
|
|
}
|
|
return {
|
|
"key": "release-evidence-flow-covers-first-class-reports",
|
|
"label": "Release evidence flow covers first-class reports",
|
|
"status": "pass" if expected == actual else "fail",
|
|
"expected": expected,
|
|
"actual": actual,
|
|
"paths": [
|
|
"AGENTS.md",
|
|
"reports/output_execution_runs.json",
|
|
"reports/install_simulation.json",
|
|
"reports/security_trust_report.json",
|
|
"reports/registry_audit.json",
|
|
"reports/package_verification.json",
|
|
"reports/upgrade_check.json",
|
|
"reports/adoption_drift_report.json",
|
|
"reports/architecture_maintainability.json",
|
|
"reports/python_compatibility.json",
|
|
"reports/runtime_permission_probes.json",
|
|
"reports/review_waivers.json",
|
|
"reports/review_annotations.json",
|
|
"reports/skill_atlas.json",
|
|
"reports/skill_os2_audit.json",
|
|
"reports/skill_os2_coverage.json",
|
|
"reports/context_budget.json",
|
|
"reports/context_budget_summary.json",
|
|
"reports/benchmark_reproducibility.json",
|
|
"reports/skill-overview.json",
|
|
"reports/skill-interpretation.json",
|
|
"reports/review-viewer.json",
|
|
"reports/world_class_evidence_preflight.json",
|
|
"reports/review-studio.json",
|
|
"reports/evidence_consistency.json",
|
|
],
|
|
"detail": (
|
|
"Release refresh and clean-lock instructions must regenerate every first-class report "
|
|
"before evidence consistency can be trusted."
|
|
),
|
|
}
|