From 965e91043b872e15cdb17ca65092fc3dcb84a273 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 16 Jun 2026 22:56:04 +0800 Subject: [PATCH] Tighten provider evidence run validation --- reports/architecture_maintainability.json | 24 ++-- reports/architecture_maintainability.md | 12 +- reports/context_budget.json | 20 +-- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 12 +- reports/python_compatibility.json | 8 +- reports/python_compatibility.md | 2 +- reports/security_trust_report.json | 30 ++++- reports/security_trust_report.md | 11 +- reports/skill-os-2-review.md | 5 +- reports/skill_os2_audit.json | 2 +- reports/skill_os2_audit.md | 2 +- reports/skill_os2_coverage.json | 2 +- reports/skill_os2_coverage.md | 2 +- scripts/run_output_execution.py | 2 +- scripts/world_class_evidence_contract.py | 26 +--- scripts/world_class_provider_evidence.py | 132 ++++++++++++++++++++ tests/verify_output_execution_runs.py | 42 +++++++ tests/verify_world_class_evidence_intake.py | 35 ++++++ tests/verify_world_class_evidence_ledger.py | 51 +++++--- 20 files changed, 330 insertions(+), 92 deletions(-) create mode 100644 scripts/world_class_provider_evidence.py diff --git a/reports/architecture_maintainability.json b/reports/architecture_maintainability.json index a20776fc..43d1a3cb 100644 --- a/reports/architecture_maintainability.json +++ b/reports/architecture_maintainability.json @@ -4,10 +4,10 @@ "generated_at": "2026-06-16", "skill_dir": ".", "summary": { - "python_file_count": 208, - "script_file_count": 135, + "python_file_count": 209, + "script_file_count": 136, "test_file_count": 73, - "internal_module_count": 51, + "internal_module_count": 52, "cli_script_count": 86, "command_handler_count": 68, "entrypoint_command_handler_count": 18, @@ -15,16 +15,23 @@ "warn_line_threshold": 900, "watch_line_threshold": 720, "block_line_threshold": 1500, - "largest_file_lines": 717, + "largest_file_lines": 704, "watchlist_count": 0, "hotspot_count": 0, "blocker_count": 0, "decision": "pass" }, "largest_files": [ + { + "path": "tests/verify_world_class_evidence_intake.py", + "lines": 704, + "kind": "test", + "severity": "pass", + "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." + }, { "path": "scripts/world_class_evidence_contract.py", - "lines": 717, + "lines": 699, "kind": "internal-module", "severity": "pass", "recommendation": "Watch this file before adding new responsibilities; extract a helper module when one concern dominates." @@ -43,13 +50,6 @@ "severity": "pass", "recommendation": "Split viewer data assembly from HTML section rendering." }, - { - "path": "tests/verify_world_class_evidence_intake.py", - "lines": 669, - "kind": "test", - "severity": "pass", - "recommendation": "Break broad integration assertions into focused verifier helpers when the next behavior change lands." - }, { "path": "scripts/skill_report_model.py", "lines": 665, diff --git a/reports/architecture_maintainability.md b/reports/architecture_maintainability.md index 841b0893..026a9923 100644 --- a/reports/architecture_maintainability.md +++ b/reports/architecture_maintainability.md @@ -5,15 +5,15 @@ Generated at: `2026-06-16` ## Summary - decision: `pass` -- python files: `208` -- scripts: `135` +- python files: `209` +- scripts: `136` - tests: `73` -- internal modules: `51` +- internal modules: `52` - CLI scripts: `86` - Yao CLI command handlers: `68` - entrypoint command handlers: `18` - command modules: `6` -- largest file lines: `717` +- largest file lines: `704` - watch threshold lines: `720` - watchlist: `0` - hotspots: `0` @@ -33,10 +33,10 @@ No near-threshold files found. | File | Lines | Kind | Severity | | --- | ---: | --- | --- | -| `scripts/world_class_evidence_contract.py` | `717` | `internal-module` | `pass` | +| `tests/verify_world_class_evidence_intake.py` | `704` | `test` | `pass` | +| `scripts/world_class_evidence_contract.py` | `699` | `internal-module` | `pass` | | `tests/verify_yao_cli.py` | `696` | `test` | `pass` | | `scripts/render_review_viewer.py` | `685` | `cli-script` | `pass` | -| `tests/verify_world_class_evidence_intake.py` | `669` | `test` | `pass` | | `scripts/skill_report_model.py` | `665` | `internal-module` | `pass` | | `scripts/render_skill_os2_coverage.py` | `649` | `cli-script` | `pass` | | `scripts/render_review_studio.py` | `647` | `cli-script` | `pass` | diff --git a/reports/context_budget.json b/reports/context_budget.json index a124eb78..e21c7b9f 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,16 +6,16 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 797, - "other_text_tokens": 1058565, + "other_text_tokens": 1060995, "estimated_initial_load_tokens": 990, - "estimated_total_text_tokens": 1059362, - "deferred_resource_tokens": 490155, + "estimated_total_text_tokens": 1061792, + "deferred_resource_tokens": 491281, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 430057, - "file_count": 135 + "estimated_tokens": 431183, + "file_count": 136 }, { "path": "references", @@ -36,8 +36,8 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 430057, - "file_count": 135 + "estimated_tokens": 431183, + "file_count": 136 } ], "deferred_resource_governance": { @@ -59,14 +59,14 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 430057, - "file_count": 135, + "estimated_tokens": 431183, + "file_count": 136, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], "summary": "Large deferred resources are indexed and backed by evidence." }, - "relevant_file_count": 663, + "relevant_file_count": 665, "unused_resource_dirs": [], "quality_signal_points": 130, "quality_density": 131.3 diff --git a/reports/context_budget.md b/reports/context_budget.md index d7b89953..69db51a6 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 990 | 797 | 490155 | `governed` | scripts:430057 | 131.3 | - | ok | +| root | `.` | `production` | 1000 | 990 | 797 | 491281 | `governed` | scripts:431183 | 131.3 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 93224ce0..b6d99af6 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -1,5 +1,5 @@ { - "generated_at": "2026-03-31", + "generated_at": "2026-06-16", "targets": [ { "label": "root", @@ -8,12 +8,12 @@ "budget_limit": 1000, "initial_tokens": 990, "skill_body_tokens": 797, - "deferred_resource_tokens": 490155, + "deferred_resource_tokens": 491281, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 430057, - "file_count": 135 + "estimated_tokens": 431183, + "file_count": 136 } ], "deferred_resource_governance": { @@ -35,8 +35,8 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 430057, - "file_count": 135, + "estimated_tokens": 431183, + "file_count": 136, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } ], diff --git a/reports/python_compatibility.json b/reports/python_compatibility.json index 5d345878..4b834f26 100644 --- a/reports/python_compatibility.json +++ b/reports/python_compatibility.json @@ -5,7 +5,7 @@ "root": ".", "summary": { "target_python": "3.11", - "file_count": 211, + "file_count": 212, "issue_count": 0, "syntax_error_count": 0, "fstring_311_violation_count": 0, @@ -760,6 +760,12 @@ "issue_count": 0, "issues": [] }, + { + "path": "scripts/world_class_provider_evidence.py", + "ok": true, + "issue_count": 0, + "issues": [] + }, { "path": "scripts/world_class_source_checks.py", "ok": true, diff --git a/reports/python_compatibility.md b/reports/python_compatibility.md index 332c4a85..357eb04a 100644 --- a/reports/python_compatibility.md +++ b/reports/python_compatibility.md @@ -6,7 +6,7 @@ Generated at: `2026-06-16` - decision: `pass` - target python: `3.11` -- files scanned: `211` +- files scanned: `212` - issues: `0` - syntax errors: `0` - f-string 3.11 violations: `0` diff --git a/reports/security_trust_report.json b/reports/security_trust_report.json index 13d2453d..0719babd 100644 --- a/reports/security_trust_report.json +++ b/reports/security_trust_report.json @@ -2,9 +2,9 @@ "ok": true, "skill_dir": ".", "summary": { - "scanned_files": 226, - "script_count": 135, - "internal_module_count": 49, + "scanned_files": 227, + "script_count": 136, + "internal_module_count": 50, "secret_findings": 0, "dependency_files": [ "requirements-ci.txt" @@ -22,8 +22,8 @@ "help_smoke_failed_count": 0, "interactive_script_count": 0, "package_hash_scope": "source-contract-without-generated-reports", - "package_hash_file_count": 226, - "package_sha256": "6149b95fa5fb9a93108ed5099f3f53e629fa6ac673e03206371d5837a31bae97" + "package_hash_file_count": 227, + "package_sha256": "01c09d5fa7c971a657e4d138499ac8d50095ae1a078c2bff1bd9b70657199c9f" }, "failures": [], "warnings": [], @@ -1726,6 +1726,20 @@ "network_urls": [], "network_hosts": [] }, + { + "path": "scripts/world_class_provider_evidence.py", + "interface": "internal-module", + "interface_declared": true, + "interface_reason": "Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows.", + "has_argparse": false, + "has_main_guard": false, + "uses_input": false, + "uses_network": false, + "uses_file_write": false, + "uses_subprocess": false, + "network_urls": [], + "network_hosts": [] + }, { "path": "scripts/world_class_source_checks.py", "interface": "internal-module", @@ -1971,7 +1985,7 @@ "checked_count": 86, "passed_count": 86, "failed_count": 0, - "skipped_count": 49, + "skipped_count": 50, "failed_scripts": [], "results": [ { @@ -2976,6 +2990,10 @@ "path": "scripts/world_class_preflight_layout.py", "reason": "internal module" }, + { + "path": "scripts/world_class_provider_evidence.py", + "reason": "internal module" + }, { "path": "scripts/world_class_source_checks.py", "reason": "internal module" diff --git a/reports/security_trust_report.md b/reports/security_trust_report.md index 7f5c83ef..06caf9ec 100644 --- a/reports/security_trust_report.md +++ b/reports/security_trust_report.md @@ -1,9 +1,9 @@ # Security Trust Report - OK: `True` -- Scanned files: `226` -- Scripts: `135` -- Internal script modules: `49` +- Scanned files: `227` +- Scripts: `136` +- Internal script modules: `50` - Secret findings: `0` - Network-capable scripts: `3` - Network policy covered scripts: `3` @@ -15,8 +15,8 @@ - CLI help smoke failures: `0` - Interactive scripts: `0` - Package hash scope: `source-contract-without-generated-reports` -- Package hash files: `226` -- Package SHA256: `6149b95fa5fb9a93108ed5099f3f53e629fa6ac673e03206371d5837a31bae97` +- Package hash files: `227` +- Package SHA256: `01c09d5fa7c971a657e4d138499ac8d50095ae1a078c2bff1bd9b70657199c9f` ## Failures @@ -182,6 +182,7 @@ | scripts/verify_package.py | cli | False | True | True | False | False | True | False | Default CLI classification; add SCRIPT_INTERFACE for internal modules. | | scripts/world_class_evidence_contract.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to share intake validation and artifact integrity checks. | | scripts/world_class_preflight_layout.py | internal-module | True | False | False | False | False | False | False | Imported by render_world_class_preflight.py to keep preflight HTML layout out of data assembly. | +| scripts/world_class_provider_evidence.py | internal-module | True | False | False | False | False | False | False | Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows. | | scripts/world_class_source_checks.py | internal-module | True | False | False | False | False | False | False | Imported by world-class evidence reports to keep source-evidence readiness checks consistent. | | scripts/world_class_submission_kit_rendering.py | internal-module | True | False | False | False | False | False | False | Shared renderer for world-class submission kit Markdown and HTML artifacts. | | scripts/world_class_submission_matrix.py | internal-module | True | False | False | False | False | False | False | Shared by submission kit rendering to summarize draft, artifact, and source-check readiness. | diff --git a/reports/skill-os-2-review.md b/reports/skill-os-2-review.md index 35209058..9d762b88 100644 --- a/reports/skill-os-2-review.md +++ b/reports/skill-os-2-review.md @@ -21,6 +21,7 @@ Yao Meta Skill is no longer only a Meta Skill factory. The current working tree - World-Class Evidence Plan v0 that turns the remaining provider holdout, human adjudication, native permission enforcement, and native client telemetry gaps into executable runbooks with success checks and privacy contracts. - World-Class Evidence Ledger v0 that records current acceptance state, provenance requirements, privacy contracts, and anti-overclaim guards for those remaining gaps. - World-Class Intake Contract Hardening v0 so real evidence submissions must use the ledger's canonical `.json` filename and are recursively rejected when they include raw prompt, output, transcript, message, credential, secret, token, or API-key fields. +- World-Class Provider Evidence Guard v0 so provider holdout submissions must reconcile `summary` counts with `runs` rows and include at least one passing model run whose provider, model, timing, non-estimated usage, and output hash match the submitted provenance. - World-Class Submission Matrix v0 so the submission kit exposes a single operator-facing matrix for draft status, artifact readiness, source-check blockers, and the next action without counting matrix rows as completion evidence. - World-Class Submission Artifact Roles v0 so submission kits distinguish `submission-ref` rows that belong in `artifact_refs` from supporting audit assets, reducing operator ambiguity without accepting evidence. - World-Class Submission Kit Rendering v0 so Markdown and HTML kit presentation lives in a dedicated internal renderer, keeping the CLI focused on evidence assembly and file emission. @@ -66,7 +67,7 @@ This is still not the final world-class state. Target-native behavior contracts | Skill OS 2.0 Audit | `scripts/render_skill_os2_audit.py`, `reports/skill_os2_audit.md`, `tests/verify_skill_os2_audit.py` | v0 landed | | World-Class Evidence Plan | `scripts/render_world_class_evidence_plan.py`, `reports/world_class_evidence_plan.md`, `tests/verify_world_class_evidence_plan.py` | v0 landed | | World-Class Evidence Ledger | `scripts/render_world_class_evidence_ledger.py`, `reports/world_class_evidence_ledger.md`, `tests/verify_world_class_evidence_ledger.py` | v0 landed | -| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, and nested raw-field rejection | v0 landed | +| World-Class Evidence Intake | `scripts/world_class_evidence_contract.py`, `scripts/world_class_provider_evidence.py`, `scripts/render_world_class_evidence_intake.py`, `evidence/world_class/intake.schema.json`, `tests/verify_world_class_evidence_intake.py` with canonical filename, source-artifact validation, provider run-row validation, and nested raw-field rejection | v0 landed | | World-Class Submission Kit | `scripts/prepare_world_class_submission_kit.py`, `scripts/world_class_submission_matrix.py`, `scripts/world_class_submission_kit_rendering.py`, `tests/verify_world_class_submission_kit.py` with draft, artifact, source-check, next-action matrix evidence, and separated Markdown/HTML rendering | v0 landed | | Runtime Conformance | `scripts/run_conformance_suite.py`, `reports/conformance_matrix.md` | v0 landed | | Trust & Security | `scripts/trust_check.py`, `reports/security_trust_report.md`, `security/*.md` | v0 landed | @@ -127,7 +128,7 @@ Next move: add real client or installer permission enforcement integration. | Output Eval | `5` cases, with-skill pass rate `100`, baseline pass rate `0`, with file-backed, near-neighbor, boundary coverage, `10` local command-runner execution runs, `0` recorded fixture runs, `0` provider model-executed runs in root release evidence, `10` estimated token counts, provider runner v0 available, `5` blind A/B review pairs, a generated `reports/output_review_decisions.json` template, `0 / 5` reviewer decisions pending, `0` answer keys revealed, and `5` pending answers hidden | | Runtime Conformance | `5 / 5` targets passing | | Target Compiler | `5 / 5` compiled target contracts generated for OpenAI, Claude, generic, Agent Skills compatible, and VS Code / Copilot outputs, including target permission contracts and target-native behavior contracts | -| Trust | `0` secret findings, `1` pinned dependency file, `49` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `135` scripts, source-contract hash scope explicit | +| Trust | `0` secret findings, `1` pinned dependency file, `50` declared internal modules, `3 / 3` network-capable scripts covered by bounded host policy, `86 / 86` CLI help smoke checks passing across `136` scripts, source-contract hash scope explicit | | Permission Governance | `3 / 3` required high-permission capabilities approved, `0` missing, `0` invalid, `0` expired | | Runtime Permission Probes | `4 / 4` target adapters probed, `0` native-enforcement adapters, `4` explicit metadata fallbacks, `4` residual risks retained for reviewer visibility | | Skill Atlas | `12` scanned skills, `1` actionable root skill, `1` telemetry report, `0` actionable route collisions, `0` actionable owner gaps, `0` actionable stale skills, `0` actionable drift signals, `24` scoped non-actionable issue signals retained for visibility | diff --git a/reports/skill_os2_audit.json b/reports/skill_os2_audit.json index 26518eba..6b6492d3 100644 --- a/reports/skill_os2_audit.json +++ b/reports/skill_os2_audit.json @@ -197,7 +197,7 @@ "key": "trust-security", "label": "Trust Security", "status": "pass", - "current": "secrets 0; scripts 131; help failures 0", + "current": "secrets 0; scripts 136; help failures 0", "target": "Secrets, scripts, dependencies, permissions, and package hash are reviewable", "evidence": [ { diff --git a/reports/skill_os2_audit.md b/reports/skill_os2_audit.md index 87f0d44a..8aab454c 100644 --- a/reports/skill_os2_audit.md +++ b/reports/skill_os2_audit.md @@ -23,7 +23,7 @@ Generated at: `2026-06-16` | Human Adjudication | human-required | 0/5 decisions; pending 5 | Real reviewer decisions recorded before claiming output review completion | Record real A/B choices in the decision template, then regenerate adjudication. | | Benchmark Reproducibility | pass | artifacts 25; missing 0; failures 3 | Public methodology, reproducible commands, required artifacts, and failure disclosure are machine-checkable | Keep the manifest current with every benchmark, package, and release evidence change. | | Runtime Conformance | pass | 5/5 targets pass | Target package structure, metadata, relative paths, and degradation notes pass | Keep target conformance fixtures updated as platform contracts change. | -| Trust Security | pass | secrets 0; scripts 131; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | +| Trust Security | pass | secrets 0; scripts 136; help failures 0 | Secrets, scripts, dependencies, permissions, and package hash are reviewable | Keep high-permission approvals scoped, expiring, and target-mapped. | | Permission Metadata | pass | 4/4 target probes pass; metadata fallback 4; installer enforcement 4 | Packaged adapters expose explicit permission metadata, residual risks, and installer enforcement evidence when available | Preserve residual-risk notes until real native enforcement exists. | | Native Permission Enforcement | external-required | native-enforced targets 0; installer-enforced targets 4 | At least one target/client enforces approved permissions at runtime | Integrate a real target-client or external installer runtime guard before claiming native permission enforcement. | | Skill Atlas | pass | 12 skills; actionable collisions 0 | Workspace catalog, route overlap, stale/owner gaps, drift, and no-route opportunities | Feed real drift data into Atlas once client telemetry is installed. | diff --git a/reports/skill_os2_coverage.json b/reports/skill_os2_coverage.json index e834dc0c..eb8b70b2 100644 --- a/reports/skill_os2_coverage.json +++ b/reports/skill_os2_coverage.json @@ -130,7 +130,7 @@ "label": "Trust Security", "status": "pass", "objective": "Scripts, dependencies, permissions, secrets, and package hash are reviewable for team distribution.", - "current": "135 scripts; secrets 0; help failures 0", + "current": "136 scripts; secrets 0; help failures 0", "command": "python3 scripts/yao.py trust .", "test": "python3 tests/verify_trust_check.py", "evidence": [ diff --git a/reports/skill_os2_coverage.md b/reports/skill_os2_coverage.md index b7941c38..38d45db4 100644 --- a/reports/skill_os2_coverage.md +++ b/reports/skill_os2_coverage.md @@ -26,7 +26,7 @@ This report maps the Skill OS 2.0 upgrade blueprint to concrete local artifacts, | Skill IR | `pass` | schema 2.0.0; targets 5 | `python3 scripts/yao.py skill-ir .` | `python3 tests/verify_skill_ir.py` | | Output Eval Lab | `pass` | 5 cases; delta 100.0; execution 10 | `python3 scripts/yao.py output-exec . && python3 scripts/yao.py output-review .` | `python3 tests/verify_output_eval_lab.py` | | Runtime Conformance | `pass` | 5/5 targets pass | `python3 scripts/yao.py conformance .` | `python3 tests/verify_conformance_suite.py` | -| Trust Security | `pass` | 135 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | +| Trust Security | `pass` | 136 scripts; secrets 0; help failures 0 | `python3 scripts/yao.py trust .` | `python3 tests/verify_trust_check.py` | | Skill Atlas | `pass` | 12 scanned skills; actionable collisions 0 | `python3 scripts/yao.py skill-atlas --workspace-root .` | `python3 tests/verify_skill_atlas.py` | | Registry Distribution | `pass` | archive entries 658; install failures 0 | `python3 scripts/yao.py package . --platform openai --platform claude --platform generic --platform vscode --output-dir dist --zip && python3 scripts/yao.py registry-audit .` | `python3 tests/verify_registry_audit.py` | | Review Studio | `pass` | 16 gates; decision review; warnings 3 | `python3 scripts/yao.py review-studio .` | `python3 tests/verify_review_studio.py` | diff --git a/scripts/run_output_execution.py b/scripts/run_output_execution.py index aa05c506..0e623689 100644 --- a/scripts/run_output_execution.py +++ b/scripts/run_output_execution.py @@ -164,7 +164,7 @@ def command_run( execution_kind = str(payload.get("execution_kind", "command")) provider = str(payload.get("provider", "")) model = str(payload.get("model", "")) - model_executed = execution_kind == "model" or bool(model and provider) + model_executed = execution_kind == "model" and bool(model and provider) return { "case_id": request["case_id"], "variant": variant, diff --git a/scripts/world_class_evidence_contract.py b/scripts/world_class_evidence_contract.py index f8e8c358..d32a4fd4 100644 --- a/scripts/world_class_evidence_contract.py +++ b/scripts/world_class_evidence_contract.py @@ -6,6 +6,8 @@ from datetime import datetime, timezone from pathlib import Path from typing import Any +from world_class_provider_evidence import validate_provider_execution_report + SCRIPT_INTERFACE = "internal-module" SCRIPT_INTERFACE_REASON = "Imported by world-class evidence reports to share intake validation and artifact integrity checks." @@ -355,28 +357,8 @@ def validate_provider_holdout_artifacts(payload: dict[str, Any], errors: list[st execution = load_json(paths.get("reports/output_execution_runs.json", root / "__missing__")) if not execution: return - execution_summary = summary(execution) - add_error(errors, execution.get("ok") is True, "provider-holdout output execution report ok must be true") - add_error( - errors, - bool(real_int(execution_summary.get("model_executed_count")) and execution_summary["model_executed_count"] > 0), - "provider-holdout output execution summary.model_executed_count must be >0", - ) - add_error( - errors, - bool(real_int(execution_summary.get("timing_observed_count")) and execution_summary["timing_observed_count"] > 0), - "provider-holdout output execution summary.timing_observed_count must be >0", - ) - add_error( - errors, - bool(real_int(execution_summary.get("token_observed_count")) and execution_summary["token_observed_count"] > 0), - "provider-holdout output execution summary.token_observed_count must be >0", - ) - add_error( - errors, - execution_summary.get("failure_count") == 0, - "provider-holdout output execution summary.failure_count must be 0", - ) + provenance = payload.get("provenance", {}) if isinstance(payload.get("provenance", {}), dict) else {} + validate_provider_execution_report(execution, provenance, errors) def validate_human_adjudication_artifacts(payload: dict[str, Any], errors: list[str], root: Path) -> None: diff --git a/scripts/world_class_provider_evidence.py b/scripts/world_class_provider_evidence.py new file mode 100644 index 00000000..fff96f00 --- /dev/null +++ b/scripts/world_class_provider_evidence.py @@ -0,0 +1,132 @@ +#!/usr/bin/env python3 +import re +from typing import Any + + +SCRIPT_INTERFACE = "internal-module" +SCRIPT_INTERFACE_REASON = "Imported by world_class_evidence_contract.py to validate provider-backed holdout execution evidence from run rows." + +SHA256_RE = re.compile(r"^[0-9a-fA-F]{64}$") + + +def add_error(errors: list[str], condition: bool, message: str) -> None: + if not condition: + errors.append(message) + + +def real_int(value: Any) -> int | None: + return value if isinstance(value, int) and not isinstance(value, bool) else None + + +def summary(payload: dict[str, Any]) -> dict[str, Any]: + value = payload.get("summary", {}) + return value if isinstance(value, dict) else {} + + +def normalized(value: Any) -> str: + return str(value or "").strip().casefold() + + +def run_rows(execution: dict[str, Any]) -> list[dict[str, Any]]: + runs = execution.get("runs", []) + return [item for item in runs if isinstance(item, dict)] if isinstance(runs, list) else [] + + +def positive_duration(value: Any) -> bool: + return isinstance(value, (int, float)) and not isinstance(value, bool) and value > 0 + + +def observed_usage(run: dict[str, Any]) -> bool: + usage = run.get("usage", {}) + if not isinstance(usage, dict) or usage.get("estimated") is not False: + return False + token_values = [ + real_int(usage.get("input_tokens")), + real_int(usage.get("output_tokens")), + real_int(usage.get("total_tokens")), + ] + return all(value is not None and value > 0 for value in token_values) + + +def provider_model_runs(execution: dict[str, Any], provenance: dict[str, Any]) -> list[dict[str, Any]]: + expected_provider = normalized(provenance.get("provider")) + expected_model = normalized(provenance.get("model")) + matches: list[dict[str, Any]] = [] + for run in run_rows(execution): + if run.get("status") != "pass": + continue + if run.get("model_executed") is not True or run.get("execution_mode") != "model": + continue + if normalized(run.get("provider")) != expected_provider: + continue + if normalized(run.get("model")) != expected_model: + continue + if not observed_usage(run): + continue + if not positive_duration(run.get("duration_ms")): + continue + if not SHA256_RE.match(str(run.get("output_sha256", "")).strip()): + continue + matches.append(run) + return matches + + +def validate_provider_execution_report( + execution: dict[str, Any], + provenance: dict[str, Any], + errors: list[str], +) -> None: + execution_summary = summary(execution) + runs = run_rows(execution) + add_error(errors, execution.get("ok") is True, "provider-holdout output execution report ok must be true") + add_error(errors, bool(runs), "provider-holdout output execution report must include run rows") + + computed_model_runs = sum(1 for run in runs if run.get("model_executed") is True) + computed_timing = sum(1 for run in runs if run.get("duration_ms") is not None) + computed_token_observed = sum(1 for run in runs if isinstance(run.get("usage"), dict) and run["usage"].get("estimated") is False) + computed_failures = sum(1 for run in runs if run.get("status") != "pass") + add_error( + errors, + real_int(execution_summary.get("model_executed_count")) == computed_model_runs, + "provider-holdout output execution summary.model_executed_count must match model-executed run rows", + ) + add_error( + errors, + real_int(execution_summary.get("timing_observed_count")) == computed_timing, + "provider-holdout output execution summary.timing_observed_count must match timed run rows", + ) + add_error( + errors, + real_int(execution_summary.get("token_observed_count")) == computed_token_observed, + "provider-holdout output execution summary.token_observed_count must match non-estimated usage rows", + ) + add_error( + errors, + real_int(execution_summary.get("failure_count")) == computed_failures, + "provider-holdout output execution summary.failure_count must match failed run rows", + ) + add_error( + errors, + bool(real_int(execution_summary.get("model_executed_count")) and execution_summary["model_executed_count"] > 0), + "provider-holdout output execution summary.model_executed_count must be >0", + ) + add_error( + errors, + bool(real_int(execution_summary.get("timing_observed_count")) and execution_summary["timing_observed_count"] > 0), + "provider-holdout output execution summary.timing_observed_count must be >0", + ) + add_error( + errors, + bool(real_int(execution_summary.get("token_observed_count")) and execution_summary["token_observed_count"] > 0), + "provider-holdout output execution summary.token_observed_count must be >0", + ) + add_error( + errors, + execution_summary.get("failure_count") == 0, + "provider-holdout output execution summary.failure_count must be 0", + ) + add_error( + errors, + bool(provider_model_runs(execution, provenance)), + "provider-holdout output execution runs must include a passing model run with matching provider, model, timing, non-estimated usage, and output hash", + ) diff --git a/tests/verify_output_execution_runs.py b/tests/verify_output_execution_runs.py index b8c73559..b2125e5a 100644 --- a/tests/verify_output_execution_runs.py +++ b/tests/verify_output_execution_runs.py @@ -129,6 +129,48 @@ def main() -> None: assert all(item["duration_ms"] is not None for item in command["runs"]), command assert all(item["model"] == "fixture-model" for item in command["runs"]), command + metadata_only_runner = tmp_root / "metadata_only_runner.py" + metadata_only_runner.write_text( + "\n".join( + [ + "import json, sys", + "request = json.loads(sys.stdin.read())", + "print(json.dumps({", + " 'output': request['fixture_output'],", + " 'provider': 'metadata-only-provider',", + " 'model': 'metadata-only-model',", + " 'usage': {'input_tokens': 11, 'output_tokens': 17, 'total_tokens': 28}", + "}))", + ] + ) + + "\n", + encoding="utf-8", + ) + metadata_only_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + "--cases", + str(ROOT / "evals" / "output" / "cases.jsonl"), + "--output-json", + str(tmp_root / "metadata-only.json"), + "--output-md", + str(tmp_root / "metadata-only.md"), + "--runner-command", + json.dumps([sys.executable, str(metadata_only_runner)]), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + metadata_only = json.loads(metadata_only_proc.stdout) + assert metadata_only["ok"], metadata_only + assert metadata_only["summary"]["command_executed_count"] == 10, metadata_only + assert metadata_only["summary"]["model_executed_count"] == 0, metadata_only + assert metadata_only["summary"]["token_observed_count"] == 10, metadata_only + assert all(item["execution_mode"] == "command" for item in metadata_only["runs"]), metadata_only + provider_request = { "case_id": "provider-contract", "variant": "with_skill", diff --git a/tests/verify_world_class_evidence_intake.py b/tests/verify_world_class_evidence_intake.py index 5e0af4e2..b438838e 100644 --- a/tests/verify_world_class_evidence_intake.py +++ b/tests/verify_world_class_evidence_intake.py @@ -147,6 +147,25 @@ def external_submission( def write_provider_artifact(skill_root: Path, *, complete: bool) -> None: + run = { + "case_id": "provider-holdout", + "variant": "with_skill", + "status": "pass", + "execution_mode": "model" if complete else "command", + "model_executed": complete, + "command_executed": True, + "duration_ms": 1234.5, + "provider": "openai" if complete else "local-output-eval-runner", + "model": "gpt-4.1-mini" if complete else "", + "usage": { + "input_tokens": 20, + "output_tokens": 30, + "total_tokens": 50, + "estimated": not complete, + }, + "output_sha256": "a" * 64, + "failure": "", + } write_json( skill_root / "reports" / "output_execution_runs.json", { @@ -159,6 +178,7 @@ def write_provider_artifact(skill_root: Path, *, complete: bool) -> None: "token_observed_count": 2 if complete else 0, "failure_count": 0, }, + "runs": [run, {**run, "case_id": "provider-holdout-2"}] if complete else [run], }, ) @@ -468,6 +488,21 @@ def assert_external_contract_artifact_validation() -> None: assert any("summary.model_executed_count must be >0" in error for error in provider_invalid["errors"]), provider_invalid["errors"] assert any("summary.token_observed_count must be >0" in error for error in provider_invalid["errors"]), provider_invalid["errors"] + write_provider_artifact(skill_root, complete=True) + forged = json.loads((skill_root / "reports" / "output_execution_runs.json").read_text(encoding="utf-8")) + for run in forged["runs"]: + run["provider"] = "local-output-eval-runner" + write_json(skill_root / "reports" / "output_execution_runs.json", forged) + forged_provider = validate_payload( + provider_artifact_submission(skill_root), + provider_entry, + path=skill_root / "evidence" / "world_class" / "submissions" / "provider-holdout.json", + root=skill_root, + template_expected=False, + ) + assert forged_provider["status"] == "fail", forged_provider + assert any("matching provider, model, timing" in error for error in forged_provider["errors"]), forged_provider["errors"] + permission_entry = {"key": "native-permission-enforcement", "category": "external"} write_native_permission_artifacts(skill_root, complete=True) permission_valid = validate_payload( diff --git a/tests/verify_world_class_evidence_ledger.py b/tests/verify_world_class_evidence_ledger.py index b15c4877..8927c650 100644 --- a/tests/verify_world_class_evidence_ledger.py +++ b/tests/verify_world_class_evidence_ledger.py @@ -67,6 +67,41 @@ def provider_submission(artifact_root: Path = ROOT, artifact_path: str = "report } +def accepted_provider_execution_report() -> dict: + return { + "schema_version": "1.0", + "ok": True, + "summary": { + "variant_run_count": 1, + "model_executed_count": 1, + "timing_observed_count": 1, + "token_observed_count": 1, + "failure_count": 0, + }, + "runs": [ + { + "case_id": "provider-holdout", + "variant": "with_skill", + "status": "pass", + "execution_mode": "model", + "model_executed": True, + "command_executed": True, + "duration_ms": 1234.5, + "provider": "openai", + "model": "gpt-4.1-mini", + "usage": { + "input_tokens": 20, + "output_tokens": 30, + "total_tokens": 50, + "estimated": False, + }, + "output_sha256": "a" * 64, + "failure": "", + } + ], + } + + def main() -> None: shutil.rmtree(TMP, ignore_errors=True) TMP.mkdir(parents=True, exist_ok=True) @@ -200,21 +235,7 @@ def main() -> None: accepted_source_skill = TMP / "accepted_source_skill" (accepted_source_skill / "reports").mkdir(parents=True) (accepted_source_skill / "reports" / "output_execution_runs.json").write_text( - json.dumps( - { - "schema_version": "1.0", - "ok": True, - "summary": { - "variant_run_count": 1, - "model_executed_count": 1, - "timing_observed_count": 1, - "token_observed_count": 1, - "failure_count": 0, - } - }, - ensure_ascii=False, - indent=2, - ) + json.dumps(accepted_provider_execution_report(), ensure_ascii=False, indent=2) + "\n", encoding="utf-8", )