From 2df5c9acd093f45e27318bbe5ff984fae3a5fd8e Mon Sep 17 00:00:00 2001 From: yaojingang Date: Mon, 15 Jun 2026 22:18:13 +0800 Subject: [PATCH] Allow conformance checks in renamed checkouts --- reports/adoption_drift_report.json | 2 +- reports/benchmark_reproducibility.json | 44 +++++++++++++++++--------- reports/benchmark_reproducibility.md | 21 ++++++------ reports/context_budget.json | 12 +++---- reports/context_budget.md | 2 +- reports/context_budget_summary.json | 6 ++-- reports/evidence_consistency.json | 28 ++++++++-------- reports/output_execution_runs.json | 20 ++++++------ reports/output_execution_runs.md | 20 ++++++------ reports/review-viewer.json | 6 ++-- reports/skill-interpretation.json | 12 +++---- reports/skill-overview.json | 12 +++---- scripts/run_conformance_suite.py | 6 +++- tests/verify_conformance_suite.py | 30 ++++++++++++++++++ 14 files changed, 135 insertions(+), 86 deletions(-) diff --git a/reports/adoption_drift_report.json b/reports/adoption_drift_report.json index 6c2e3e4..2d9d86f 100644 --- a/reports/adoption_drift_report.json +++ b/reports/adoption_drift_report.json @@ -1,7 +1,7 @@ { "ok": true, "schema_version": "2.0", - "generated_at": "2026-06-15T14:15:32Z", + "generated_at": "2026-06-15T14:17:52Z", "skill_dir": ".", "privacy_contract": { "storage": "local-first", diff --git a/reports/benchmark_reproducibility.json b/reports/benchmark_reproducibility.json index f8cc35e..aaa43f4 100644 --- a/reports/benchmark_reproducibility.json +++ b/reports/benchmark_reproducibility.json @@ -3,21 +3,34 @@ "ok": true, "generated_at": "2026-06-15", "skill_dir": ".", - "commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6", + "commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", "git_status": { "available": true, - "dirty": false, - "changed_file_count": 0, - "sample": [], + "dirty": true, + "changed_file_count": 12, + "sample": [ + " M reports/adoption_drift_report.json", + " M reports/benchmark_reproducibility.json", + " M reports/benchmark_reproducibility.md", + " M reports/context_budget.json", + " M reports/context_budget.md", + " M reports/context_budget_summary.json", + " M reports/output_execution_runs.json", + " M reports/output_execution_runs.md", + " M reports/review-viewer.json", + " M reports/skill-overview.json", + " M scripts/run_conformance_suite.py", + " M tests/verify_conformance_suite.py" + ], "scope": "generation-time status before this report is written" }, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab", + "evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc", "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, @@ -37,14 +50,15 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 12 }, "public_claim": { "ready": false, "scope": "public benchmark or world-class readiness claim", "blockers": [ + "release lock is not clean or commit is unavailable", "provider-backed model holdout evidence is incomplete", "human blind-review adjudication is incomplete", "world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)", @@ -53,10 +67,10 @@ "policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks." }, "release_lock": { - "ready": true, - "commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6", + "ready": false, + "commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", "status_scope": "generation-time status before this report is written", - "reason": "clean generation-time HEAD" + "reason": "working tree was dirty at generation time" }, "evidence_bundle": { "algorithm": "sha256(path,label,exists,artifact_sha256)", @@ -64,7 +78,7 @@ "existing_count": 24, "missing_count": 0, "missing_paths": [], - "sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab" + "sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc" }, "methodology": { "path": "reports/benchmark_methodology.md", @@ -137,8 +151,8 @@ "label": "output_execution", "path": "reports/output_execution_runs.json", "exists": true, - "bytes": 7965, - "sha256": "3f13fd99af6576302e9fffc8ef78161b2f9a7ef95bf94a22d361a3cb7bda2843" + "bytes": 7967, + "sha256": "cec7afbfb8daa00e5fcfb3c79dac10163e312e307431896cc0f62b0f47416659" }, { "label": "blind_review", diff --git a/reports/benchmark_reproducibility.md b/reports/benchmark_reproducibility.md index c0cf267..c845b1f 100644 --- a/reports/benchmark_reproducibility.md +++ b/reports/benchmark_reproducibility.md @@ -1,14 +1,14 @@ # Benchmark Reproducibility Generated at: `2026-06-15` -Commit: `27a5c120081fe8f2518de60c6a2278f41762e9b6` -Working tree dirty at generation: `false` -Evidence bundle SHA256: `0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab` +Commit: `6e2dfd48d080901f09b4e67bdf35b9500ec3dcec` +Working tree dirty at generation: `true` +Evidence bundle SHA256: `4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc` ## Summary - reproducibility ready: `true` -- release lock ready: `true` +- release lock ready: `false` - methodology complete: `true` - required artifacts: `24` - missing artifacts: `0` @@ -22,8 +22,8 @@ Evidence bundle SHA256: `0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a52 - world-class ready: `false` - world-class source checks: `6` pass / `13` total; `7` blocked - public claim ready: `false` -- public claim blockers: `4` -- changed files at generation: `0` +- public claim blockers: `5` +- changed files at generation: `12` This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below. @@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | Blocker | | --- | +| release lock is not clean or commit is unavailable | | provider-backed model holdout evidence is incomplete | | human blind-review adjudication is incomplete | | world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) | @@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi ## Release Lock -- ready: `true` -- reason: clean generation-time HEAD +- ready: `false` +- reason: working tree was dirty at generation time - status scope: generation-time status before this report is written ## Evidence Bundle - algorithm: `sha256(path,label,exists,artifact_sha256)` - artifacts: `24` / `24` -- sha256: `0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab` +- sha256: `4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc` ## Methodology Sections @@ -72,7 +73,7 @@ This report proves local benchmark reproducibility only. It keeps external provi | output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` | | output_schema | `evals/output/schema.json` | present | `8ee340c95064` | | output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` | -| output_execution | `reports/output_execution_runs.json` | present | `3f13fd99af65` | +| output_execution | `reports/output_execution_runs.json` | present | `cec7afbfb8da` | | blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` | | review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` | | trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` | diff --git a/reports/context_budget.json b/reports/context_budget.json index a8bcb18..f508710 100644 --- a/reports/context_budget.json +++ b/reports/context_budget.json @@ -6,15 +6,15 @@ "context_budget_tier": "production", "context_budget_limit": 1000, "skill_body_tokens": 767, - "other_text_tokens": 1333919, + "other_text_tokens": 1334221, "estimated_initial_load_tokens": 960, - "estimated_total_text_tokens": 1334686, - "deferred_resource_tokens": 421781, + "estimated_total_text_tokens": 1334988, + "deferred_resource_tokens": 421829, "deferred_resource_warn_threshold": 120000, "deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 373191, + "estimated_tokens": 373239, "file_count": 107 }, { @@ -31,7 +31,7 @@ "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 373191, + "estimated_tokens": 373239, "file_count": 107 } ], @@ -54,7 +54,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 373191, + "estimated_tokens": 373239, "file_count": 107, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/context_budget.md b/reports/context_budget.md index ac4e011..8f365dd 100644 --- a/reports/context_budget.md +++ b/reports/context_budget.md @@ -2,7 +2,7 @@ | Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status | | --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- | -| root | `.` | `production` | 1000 | 960 | 767 | 421781 | `governed` | scripts:373191 | 135.4 | - | ok | +| root | `.` | `production` | 1000 | 960 | 767 | 421829 | `governed` | scripts:373239 | 135.4 | - | ok | | complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok | | governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok | diff --git a/reports/context_budget_summary.json b/reports/context_budget_summary.json index 19092b3..4da74d6 100644 --- a/reports/context_budget_summary.json +++ b/reports/context_budget_summary.json @@ -8,11 +8,11 @@ "budget_limit": 1000, "initial_tokens": 960, "skill_body_tokens": 767, - "deferred_resource_tokens": 421781, + "deferred_resource_tokens": 421829, "large_deferred_resource_dirs": [ { "path": "scripts", - "estimated_tokens": 373191, + "estimated_tokens": 373239, "file_count": 107 } ], @@ -35,7 +35,7 @@ ], "missing": [], "path": "scripts", - "estimated_tokens": 373191, + "estimated_tokens": 373239, "file_count": 107, "rationale": "Script resources are deterministic deferred tools, not initial-load prompt context." } diff --git a/reports/evidence_consistency.json b/reports/evidence_consistency.json index ebf7b9b..ae3eb19 100644 --- a/reports/evidence_consistency.json +++ b/reports/evidence_consistency.json @@ -37,8 +37,8 @@ "key": "benchmark-release-lock-self-consistency", "label": "Benchmark release lock matches git dirty state", "status": "pass", - "expected": true, - "actual": true, + "expected": false, + "actual": false, "paths": [ "reports/benchmark_reproducibility.json" ], @@ -48,8 +48,8 @@ "key": "overview-benchmark-commit", "label": "overview embeds the benchmark commit", "status": "pass", - "expected": "27a5c120081fe8f2518de60c6a2278f41762e9b6", - "actual": "27a5c120081fe8f2518de60c6a2278f41762e9b6", + "expected": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", + "actual": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-overview.json" @@ -61,7 +61,7 @@ "label": "overview embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", @@ -71,10 +71,10 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", @@ -84,7 +84,7 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", @@ -194,8 +194,8 @@ "key": "interpretation-benchmark-commit", "label": "interpretation embeds the benchmark commit", "status": "pass", - "expected": "27a5c120081fe8f2518de60c6a2278f41762e9b6", - "actual": "27a5c120081fe8f2518de60c6a2278f41762e9b6", + "expected": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", + "actual": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", "paths": [ "reports/benchmark_reproducibility.json", "reports/skill-interpretation.json" @@ -207,7 +207,7 @@ "label": "interpretation embeds benchmark summary fields", "status": "pass", "expected": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", @@ -217,10 +217,10 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "actual": { - "release_lock_ready": true, + "release_lock_ready": false, "required_artifact_count": 24, "missing_artifact_count": 0, "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", @@ -230,7 +230,7 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4 + "public_claim_blocker_count": 5 }, "paths": [ "reports/benchmark_reproducibility.json", diff --git a/reports/output_execution_runs.json b/reports/output_execution_runs.json index 3079576..0bb2134 100644 --- a/reports/output_execution_runs.json +++ b/reports/output_execution_runs.json @@ -34,7 +34,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.76, + "duration_ms": 28.57, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -62,7 +62,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.9, + "duration_ms": 28.87, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -85,7 +85,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.6, + "duration_ms": 27.45, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -113,7 +113,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.21, + "duration_ms": 26.35, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -136,7 +136,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.73, + "duration_ms": 26.57, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -164,7 +164,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.14, + "duration_ms": 26.15, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -187,7 +187,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.71, + "duration_ms": 25.93, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -214,7 +214,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.58, + "duration_ms": 25.88, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -237,7 +237,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 28.49, + "duration_ms": 26.71, "provider": "local-output-eval-runner", "model": "", "usage": { @@ -266,7 +266,7 @@ "execution_mode": "command", "model_executed": false, "command_executed": true, - "duration_ms": 27.46, + "duration_ms": 26.73, "provider": "local-output-eval-runner", "model": "", "usage": { diff --git a/reports/output_execution_runs.md b/reports/output_execution_runs.md index 7b6460c..a7596c8 100644 --- a/reports/output_execution_runs.md +++ b/reports/output_execution_runs.md @@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext | Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status | | --- | --- | --- | --- | ---: | ---: | ---: | --- | -| skill-package-contract | baseline | command | local-output-eval-runner | 28.76 | 33 | 0.0 | pass | -| skill-package-contract | with_skill | command | local-output-eval-runner | 27.9 | 73 | 100.0 | pass | -| output-eval-expectation | baseline | command | local-output-eval-runner | 27.6 | 36 | 0.0 | pass | -| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.21 | 80 | 100.0 | pass | -| ir-before-packaging | baseline | command | local-output-eval-runner | 28.73 | 33 | 0.0 | pass | -| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.14 | 80 | 100.0 | pass | -| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.71 | 36 | 0.0 | pass | -| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.58 | 65 | 100.0 | pass | -| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.49 | 37 | 0.0 | pass | -| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.46 | 98 | 100.0 | pass | +| skill-package-contract | baseline | command | local-output-eval-runner | 28.57 | 33 | 0.0 | pass | +| skill-package-contract | with_skill | command | local-output-eval-runner | 28.87 | 73 | 100.0 | pass | +| output-eval-expectation | baseline | command | local-output-eval-runner | 27.45 | 36 | 0.0 | pass | +| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.35 | 80 | 100.0 | pass | +| ir-before-packaging | baseline | command | local-output-eval-runner | 26.57 | 33 | 0.0 | pass | +| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.15 | 80 | 100.0 | pass | +| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.93 | 36 | 0.0 | pass | +| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.88 | 65 | 100.0 | pass | +| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.71 | 37 | 0.0 | pass | +| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.73 | 98 | 100.0 | pass | ## Next Fixes diff --git a/reports/review-viewer.json b/reports/review-viewer.json index e5b450e..bae1199 100644 --- a/reports/review-viewer.json +++ b/reports/review-viewer.json @@ -1000,7 +1000,7 @@ "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab", + "evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc", "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, @@ -1022,9 +1022,9 @@ "public_claim_ready": false, "public_claim_blocker_count": 5, "working_tree_dirty": true, - "changed_file_count": 38 + "changed_file_count": 12 }, - "commit": "db1b24cbda5eb00f71b35846dff0f178625aa31d", + "commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/reports/skill-interpretation.json b/reports/skill-interpretation.json index d13f5d1..f3c590d 100644 --- a/reports/skill-interpretation.json +++ b/reports/skill-interpretation.json @@ -1000,11 +1000,11 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab", + "evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc", "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, @@ -1024,11 +1024,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 12 }, - "commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6", + "commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/reports/skill-overview.json b/reports/skill-overview.json index f80a7e0..74887e9 100644 --- a/reports/skill-overview.json +++ b/reports/skill-overview.json @@ -995,11 +995,11 @@ "ok": true, "summary": { "reproducibility_ready": true, - "release_lock_ready": true, + "release_lock_ready": false, "methodology_complete": true, "required_artifact_count": 24, "missing_artifact_count": 0, - "evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab", + "evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc", "source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306", "archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc", "output_case_count": 5, @@ -1019,11 +1019,11 @@ "world_class_source_pass_count": 6, "world_class_source_blocked_count": 7, "public_claim_ready": false, - "public_claim_blocker_count": 4, - "working_tree_dirty": false, - "changed_file_count": 0 + "public_claim_blocker_count": 5, + "working_tree_dirty": true, + "changed_file_count": 12 }, - "commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6", + "commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec", "missing_artifacts": [], "limitations": [ "The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.", diff --git a/scripts/run_conformance_suite.py b/scripts/run_conformance_suite.py index d51aa25..59d68ee 100644 --- a/scripts/run_conformance_suite.py +++ b/scripts/run_conformance_suite.py @@ -144,7 +144,11 @@ def check_target(skill_dir: Path, target: str, evidence: dict[str, Any]) -> dict add_check(checks, failures, re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name) is not None, "name is runtime-safe", "name contains unsafe characters") if target in {"agent-skills", "vscode"}: - add_check(checks, failures, skill_dir.name == name, "directory name matches skill name", "directory name must match skill name") + add_check(checks, failures, bool(name), "package identity derives from skill name", "Missing package identity") + if skill_dir.name != name: + warnings.append( + "source checkout directory differs from skill name; package verification must enforce archive top-level identity." + ) for field in ("name", "version", "owner", "status", "maturity_tier", "review_cadence"): add_check(checks, failures, bool(manifest.get(field)), f"manifest.{field} exists", f"Missing manifest.{field}") diff --git a/tests/verify_conformance_suite.py b/tests/verify_conformance_suite.py index 5039ee5..cd900ab 100644 --- a/tests/verify_conformance_suite.py +++ b/tests/verify_conformance_suite.py @@ -1,7 +1,9 @@ #!/usr/bin/env python3 import json +import shutil import subprocess import sys +import tempfile from pathlib import Path @@ -52,6 +54,34 @@ def main() -> None: assert output_md.exists(), output_md assert "Runtime Conformance Matrix" in output_md.read_text(encoding="utf-8") + with tempfile.TemporaryDirectory(prefix="renamed-conformance-root-") as temp_root: + renamed_root = Path(temp_root) / "checkout-alias" + shutil.copytree( + ROOT, + renamed_root, + ignore=shutil.ignore_patterns(".git", ".previews", "dist", "__pycache__", ".pytest_cache", "tmp*"), + ) + renamed_proc = subprocess.run( + [ + sys.executable, + str(SCRIPT), + str(renamed_root), + "--output-json", + str(tmp_root / "renamed_conformance_matrix.json"), + "--output-md", + str(tmp_root / "renamed_conformance_matrix.md"), + ], + cwd=ROOT, + capture_output=True, + text=True, + check=True, + ) + renamed_payload = json.loads(renamed_proc.stdout) + assert renamed_payload["ok"], renamed_payload + assert renamed_payload["summary"]["pass_count"] == 5, renamed_payload + warnings = [warning for target in renamed_payload["targets"] for warning in target["warnings"]] + assert any("source checkout directory differs" in warning for warning in warnings), renamed_payload + broken = tmp_root / "broken-skill" (broken / "agents").mkdir(parents=True, exist_ok=True) (broken / "reports").mkdir(parents=True, exist_ok=True)