Allow conformance checks in renamed checkouts
This commit is contained in:
yaojingang
@@ -1,7 +1,7 @@
|
||||
{
|
||||
"ok": true,
|
||||
"schema_version": "2.0",
|
||||
"generated_at": "2026-06-15T14:15:32Z",
|
||||
"generated_at": "2026-06-15T14:17:52Z",
|
||||
"skill_dir": ".",
|
||||
"privacy_contract": {
|
||||
"storage": "local-first",
|
||||
|
||||
@@ -3,21 +3,34 @@
|
||||
"ok": true,
|
||||
"generated_at": "2026-06-15",
|
||||
"skill_dir": ".",
|
||||
"commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"git_status": {
|
||||
"available": true,
|
||||
"dirty": false,
|
||||
"changed_file_count": 0,
|
||||
"sample": [],
|
||||
"dirty": true,
|
||||
"changed_file_count": 12,
|
||||
"sample": [
|
||||
" M reports/adoption_drift_report.json",
|
||||
" M reports/benchmark_reproducibility.json",
|
||||
" M reports/benchmark_reproducibility.md",
|
||||
" M reports/context_budget.json",
|
||||
" M reports/context_budget.md",
|
||||
" M reports/context_budget_summary.json",
|
||||
" M reports/output_execution_runs.json",
|
||||
" M reports/output_execution_runs.md",
|
||||
" M reports/review-viewer.json",
|
||||
" M reports/skill-overview.json",
|
||||
" M scripts/run_conformance_suite.py",
|
||||
" M tests/verify_conformance_suite.py"
|
||||
],
|
||||
"scope": "generation-time status before this report is written"
|
||||
},
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab",
|
||||
"evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc",
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc",
|
||||
"output_case_count": 5,
|
||||
@@ -37,14 +50,15 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 12
|
||||
},
|
||||
"public_claim": {
|
||||
"ready": false,
|
||||
"scope": "public benchmark or world-class readiness claim",
|
||||
"blockers": [
|
||||
"release lock is not clean or commit is unavailable",
|
||||
"provider-backed model holdout evidence is incomplete",
|
||||
"human blind-review adjudication is incomplete",
|
||||
"world-class evidence is not accepted yet (4 open gaps, 4 ledger pending)",
|
||||
@@ -53,10 +67,10 @@
|
||||
"policy": "Local reproducibility can pass before public claims; public claims require provider evidence, human adjudication, clean release lock, accepted world-class evidence, and complete source checks."
|
||||
},
|
||||
"release_lock": {
|
||||
"ready": true,
|
||||
"commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"ready": false,
|
||||
"commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"status_scope": "generation-time status before this report is written",
|
||||
"reason": "clean generation-time HEAD"
|
||||
"reason": "working tree was dirty at generation time"
|
||||
},
|
||||
"evidence_bundle": {
|
||||
"algorithm": "sha256(path,label,exists,artifact_sha256)",
|
||||
@@ -64,7 +78,7 @@
|
||||
"existing_count": 24,
|
||||
"missing_count": 0,
|
||||
"missing_paths": [],
|
||||
"sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab"
|
||||
"sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc"
|
||||
},
|
||||
"methodology": {
|
||||
"path": "reports/benchmark_methodology.md",
|
||||
@@ -137,8 +151,8 @@
|
||||
"label": "output_execution",
|
||||
"path": "reports/output_execution_runs.json",
|
||||
"exists": true,
|
||||
"bytes": 7965,
|
||||
"sha256": "3f13fd99af6576302e9fffc8ef78161b2f9a7ef95bf94a22d361a3cb7bda2843"
|
||||
"bytes": 7967,
|
||||
"sha256": "cec7afbfb8daa00e5fcfb3c79dac10163e312e307431896cc0f62b0f47416659"
|
||||
},
|
||||
{
|
||||
"label": "blind_review",
|
||||
|
||||
@@ -1,14 +1,14 @@
|
||||
# Benchmark Reproducibility
|
||||
|
||||
Generated at: `2026-06-15`
|
||||
Commit: `27a5c120081fe8f2518de60c6a2278f41762e9b6`
|
||||
Working tree dirty at generation: `false`
|
||||
Evidence bundle SHA256: `0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab`
|
||||
Commit: `6e2dfd48d080901f09b4e67bdf35b9500ec3dcec`
|
||||
Working tree dirty at generation: `true`
|
||||
Evidence bundle SHA256: `4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc`
|
||||
|
||||
## Summary
|
||||
|
||||
- reproducibility ready: `true`
|
||||
- release lock ready: `true`
|
||||
- release lock ready: `false`
|
||||
- methodology complete: `true`
|
||||
- required artifacts: `24`
|
||||
- missing artifacts: `0`
|
||||
@@ -22,8 +22,8 @@ Evidence bundle SHA256: `0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a52
|
||||
- world-class ready: `false`
|
||||
- world-class source checks: `6` pass / `13` total; `7` blocked
|
||||
- public claim ready: `false`
|
||||
- public claim blockers: `4`
|
||||
- changed files at generation: `0`
|
||||
- public claim blockers: `5`
|
||||
- changed files at generation: `12`
|
||||
|
||||
This report proves local benchmark reproducibility only. It keeps external provider and human-review gaps visible instead of counting them as complete. The git commit is generation-time context; the evidence bundle SHA is the durable anchor for the artifacts listed below.
|
||||
|
||||
@@ -35,6 +35,7 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
|
||||
| Blocker |
|
||||
| --- |
|
||||
| release lock is not clean or commit is unavailable |
|
||||
| provider-backed model holdout evidence is incomplete |
|
||||
| human blind-review adjudication is incomplete |
|
||||
| world-class evidence is not accepted yet (4 open gaps, 4 ledger pending) |
|
||||
@@ -42,15 +43,15 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
|
||||
## Release Lock
|
||||
|
||||
- ready: `true`
|
||||
- reason: clean generation-time HEAD
|
||||
- ready: `false`
|
||||
- reason: working tree was dirty at generation time
|
||||
- status scope: generation-time status before this report is written
|
||||
|
||||
## Evidence Bundle
|
||||
|
||||
- algorithm: `sha256(path,label,exists,artifact_sha256)`
|
||||
- artifacts: `24` / `24`
|
||||
- sha256: `0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab`
|
||||
- sha256: `4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc`
|
||||
|
||||
## Methodology Sections
|
||||
|
||||
@@ -72,7 +73,7 @@ This report proves local benchmark reproducibility only. It keeps external provi
|
||||
| output_cases | `evals/output/cases.jsonl` | present | `a6ae96857116` |
|
||||
| output_schema | `evals/output/schema.json` | present | `8ee340c95064` |
|
||||
| output_scorecard | `reports/output_quality_scorecard.json` | present | `0806258a8e08` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `3f13fd99af65` |
|
||||
| output_execution | `reports/output_execution_runs.json` | present | `cec7afbfb8da` |
|
||||
| blind_review | `reports/output_blind_review_pack.json` | present | `bbe2db8ec277` |
|
||||
| review_adjudication | `reports/output_review_adjudication.json` | present | `240485a721af` |
|
||||
| trigger_scorecard | `reports/route_scorecard.json` | present | `c164e83e36d0` |
|
||||
|
||||
@@ -6,15 +6,15 @@
|
||||
"context_budget_tier": "production",
|
||||
"context_budget_limit": 1000,
|
||||
"skill_body_tokens": 767,
|
||||
"other_text_tokens": 1333919,
|
||||
"other_text_tokens": 1334221,
|
||||
"estimated_initial_load_tokens": 960,
|
||||
"estimated_total_text_tokens": 1334686,
|
||||
"deferred_resource_tokens": 421781,
|
||||
"estimated_total_text_tokens": 1334988,
|
||||
"deferred_resource_tokens": 421829,
|
||||
"deferred_resource_warn_threshold": 120000,
|
||||
"deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 373191,
|
||||
"estimated_tokens": 373239,
|
||||
"file_count": 107
|
||||
},
|
||||
{
|
||||
@@ -31,7 +31,7 @@
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 373191,
|
||||
"estimated_tokens": 373239,
|
||||
"file_count": 107
|
||||
}
|
||||
],
|
||||
@@ -54,7 +54,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 373191,
|
||||
"estimated_tokens": 373239,
|
||||
"file_count": 107,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -2,7 +2,7 @@
|
||||
|
||||
| Target | Path | Tier | Limit | Initial | SKILL | Deferred | Resource Governance | Large Deferred Dirs | Quality Density | Unused Dirs | Status |
|
||||
| --- | --- | --- | ---: | ---: | ---: | ---: | --- | --- | ---: | --- | --- |
|
||||
| root | `.` | `production` | 1000 | 960 | 767 | 421781 | `governed` | scripts:373191 | 135.4 | - | ok |
|
||||
| root | `.` | `production` | 1000 | 960 | 767 | 421829 | `governed` | scripts:373239 | 135.4 | - | ok |
|
||||
| complex-release-orchestrator | `examples/complex-release-orchestrator/generated-skill` | `production` | 1000 | 790 | 718 | 1657 | `not-required` | - | 164.6 | - | ok |
|
||||
| governed-incident-command | `examples/governed-incident-command/generated-skill` | `production` | 1000 | 760 | 658 | 1030 | `not-required` | - | 171.1 | - | ok |
|
||||
|
||||
|
||||
@@ -8,11 +8,11 @@
|
||||
"budget_limit": 1000,
|
||||
"initial_tokens": 960,
|
||||
"skill_body_tokens": 767,
|
||||
"deferred_resource_tokens": 421781,
|
||||
"deferred_resource_tokens": 421829,
|
||||
"large_deferred_resource_dirs": [
|
||||
{
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 373191,
|
||||
"estimated_tokens": 373239,
|
||||
"file_count": 107
|
||||
}
|
||||
],
|
||||
@@ -35,7 +35,7 @@
|
||||
],
|
||||
"missing": [],
|
||||
"path": "scripts",
|
||||
"estimated_tokens": 373191,
|
||||
"estimated_tokens": 373239,
|
||||
"file_count": 107,
|
||||
"rationale": "Script resources are deterministic deferred tools, not initial-load prompt context."
|
||||
}
|
||||
|
||||
@@ -37,8 +37,8 @@
|
||||
"key": "benchmark-release-lock-self-consistency",
|
||||
"label": "Benchmark release lock matches git dirty state",
|
||||
"status": "pass",
|
||||
"expected": true,
|
||||
"actual": true,
|
||||
"expected": false,
|
||||
"actual": false,
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json"
|
||||
],
|
||||
@@ -48,8 +48,8 @@
|
||||
"key": "overview-benchmark-commit",
|
||||
"label": "overview embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"actual": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"expected": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"actual": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-overview.json"
|
||||
@@ -61,7 +61,7 @@
|
||||
"label": "overview embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
@@ -71,10 +71,10 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
@@ -84,7 +84,7 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
@@ -194,8 +194,8 @@
|
||||
"key": "interpretation-benchmark-commit",
|
||||
"label": "interpretation embeds the benchmark commit",
|
||||
"status": "pass",
|
||||
"expected": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"actual": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"expected": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"actual": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
"reports/skill-interpretation.json"
|
||||
@@ -207,7 +207,7 @@
|
||||
"label": "interpretation embeds benchmark summary fields",
|
||||
"status": "pass",
|
||||
"expected": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
@@ -217,10 +217,10 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"actual": {
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
@@ -230,7 +230,7 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4
|
||||
"public_claim_blocker_count": 5
|
||||
},
|
||||
"paths": [
|
||||
"reports/benchmark_reproducibility.json",
|
||||
|
||||
@@ -34,7 +34,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.76,
|
||||
"duration_ms": 28.57,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -62,7 +62,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.9,
|
||||
"duration_ms": 28.87,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -85,7 +85,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.6,
|
||||
"duration_ms": 27.45,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -113,7 +113,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.21,
|
||||
"duration_ms": 26.35,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -136,7 +136,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.73,
|
||||
"duration_ms": 26.57,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -164,7 +164,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.14,
|
||||
"duration_ms": 26.15,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -187,7 +187,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.71,
|
||||
"duration_ms": 25.93,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -214,7 +214,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.58,
|
||||
"duration_ms": 25.88,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -237,7 +237,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 28.49,
|
||||
"duration_ms": 26.71,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
@@ -266,7 +266,7 @@
|
||||
"execution_mode": "command",
|
||||
"model_executed": false,
|
||||
"command_executed": true,
|
||||
"duration_ms": 27.46,
|
||||
"duration_ms": 26.73,
|
||||
"provider": "local-output-eval-runner",
|
||||
"model": "",
|
||||
"usage": {
|
||||
|
||||
@@ -23,16 +23,16 @@ Command runner evidence is present. This proves the eval harness executed an ext
|
||||
|
||||
| Case | Variant | Mode | Model | Duration ms | Tokens | Score | Status |
|
||||
| --- | --- | --- | --- | ---: | ---: | ---: | --- |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 28.76 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 27.9 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.6 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 28.21 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 28.73 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 28.14 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 28.71 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 28.58 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 28.49 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 27.46 | 98 | 100.0 | pass |
|
||||
| skill-package-contract | baseline | command | local-output-eval-runner | 28.57 | 33 | 0.0 | pass |
|
||||
| skill-package-contract | with_skill | command | local-output-eval-runner | 28.87 | 73 | 100.0 | pass |
|
||||
| output-eval-expectation | baseline | command | local-output-eval-runner | 27.45 | 36 | 0.0 | pass |
|
||||
| output-eval-expectation | with_skill | command | local-output-eval-runner | 26.35 | 80 | 100.0 | pass |
|
||||
| ir-before-packaging | baseline | command | local-output-eval-runner | 26.57 | 33 | 0.0 | pass |
|
||||
| ir-before-packaging | with_skill | command | local-output-eval-runner | 26.15 | 80 | 100.0 | pass |
|
||||
| near-neighbor-boundary | baseline | command | local-output-eval-runner | 25.93 | 36 | 0.0 | pass |
|
||||
| near-neighbor-boundary | with_skill | command | local-output-eval-runner | 25.88 | 65 | 100.0 | pass |
|
||||
| file-backed-governed-package | baseline | command | local-output-eval-runner | 26.71 | 37 | 0.0 | pass |
|
||||
| file-backed-governed-package | with_skill | command | local-output-eval-runner | 26.73 | 98 | 100.0 | pass |
|
||||
|
||||
## Next Fixes
|
||||
|
||||
|
||||
@@ -1000,7 +1000,7 @@
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab",
|
||||
"evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc",
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc",
|
||||
"output_case_count": 5,
|
||||
@@ -1022,9 +1022,9 @@
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 38
|
||||
"changed_file_count": 12
|
||||
},
|
||||
"commit": "db1b24cbda5eb00f71b35846dff0f178625aa31d",
|
||||
"commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
|
||||
@@ -1000,11 +1000,11 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab",
|
||||
"evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc",
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc",
|
||||
"output_case_count": 5,
|
||||
@@ -1024,11 +1024,11 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 12
|
||||
},
|
||||
"commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
|
||||
@@ -995,11 +995,11 @@
|
||||
"ok": true,
|
||||
"summary": {
|
||||
"reproducibility_ready": true,
|
||||
"release_lock_ready": true,
|
||||
"release_lock_ready": false,
|
||||
"methodology_complete": true,
|
||||
"required_artifact_count": 24,
|
||||
"missing_artifact_count": 0,
|
||||
"evidence_bundle_sha256": "0d1c762a722d1bbc83339a0365efcfcdf027849fc83c7079da92a524b905f5ab",
|
||||
"evidence_bundle_sha256": "4b8a09b7d324ac39b25a0cc9af8ad04c85e6c94c3d1280c6243146f0f4048bcc",
|
||||
"source_contract_sha256": "4660a11db94947ab603dca42eddc447698785d08f0df2972bf2ca43454683306",
|
||||
"archive_sha256": "6852cf91a74d232c32d732b7c159c971827abf23af50153987193b084ad3b5cc",
|
||||
"output_case_count": 5,
|
||||
@@ -1019,11 +1019,11 @@
|
||||
"world_class_source_pass_count": 6,
|
||||
"world_class_source_blocked_count": 7,
|
||||
"public_claim_ready": false,
|
||||
"public_claim_blocker_count": 4,
|
||||
"working_tree_dirty": false,
|
||||
"changed_file_count": 0
|
||||
"public_claim_blocker_count": 5,
|
||||
"working_tree_dirty": true,
|
||||
"changed_file_count": 12
|
||||
},
|
||||
"commit": "27a5c120081fe8f2518de60c6a2278f41762e9b6",
|
||||
"commit": "6e2dfd48d080901f09b4e67bdf35b9500ec3dcec",
|
||||
"missing_artifacts": [],
|
||||
"limitations": [
|
||||
"The git commit and dirty flag are generation-time context; the evidence bundle hash is the durable artifact anchor inside a committed report.",
|
||||
|
||||
@@ -144,7 +144,11 @@ def check_target(skill_dir: Path, target: str, evidence: dict[str, Any]) -> dict
|
||||
add_check(checks, failures, re.fullmatch(r"[a-z0-9][a-z0-9_-]*", name) is not None, "name is runtime-safe", "name contains unsafe characters")
|
||||
|
||||
if target in {"agent-skills", "vscode"}:
|
||||
add_check(checks, failures, skill_dir.name == name, "directory name matches skill name", "directory name must match skill name")
|
||||
add_check(checks, failures, bool(name), "package identity derives from skill name", "Missing package identity")
|
||||
if skill_dir.name != name:
|
||||
warnings.append(
|
||||
"source checkout directory differs from skill name; package verification must enforce archive top-level identity."
|
||||
)
|
||||
|
||||
for field in ("name", "version", "owner", "status", "maturity_tier", "review_cadence"):
|
||||
add_check(checks, failures, bool(manifest.get(field)), f"manifest.{field} exists", f"Missing manifest.{field}")
|
||||
|
||||
@@ -1,7 +1,9 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import tempfile
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
@@ -52,6 +54,34 @@ def main() -> None:
|
||||
assert output_md.exists(), output_md
|
||||
assert "Runtime Conformance Matrix" in output_md.read_text(encoding="utf-8")
|
||||
|
||||
with tempfile.TemporaryDirectory(prefix="renamed-conformance-root-") as temp_root:
|
||||
renamed_root = Path(temp_root) / "checkout-alias"
|
||||
shutil.copytree(
|
||||
ROOT,
|
||||
renamed_root,
|
||||
ignore=shutil.ignore_patterns(".git", ".previews", "dist", "__pycache__", ".pytest_cache", "tmp*"),
|
||||
)
|
||||
renamed_proc = subprocess.run(
|
||||
[
|
||||
sys.executable,
|
||||
str(SCRIPT),
|
||||
str(renamed_root),
|
||||
"--output-json",
|
||||
str(tmp_root / "renamed_conformance_matrix.json"),
|
||||
"--output-md",
|
||||
str(tmp_root / "renamed_conformance_matrix.md"),
|
||||
],
|
||||
cwd=ROOT,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
check=True,
|
||||
)
|
||||
renamed_payload = json.loads(renamed_proc.stdout)
|
||||
assert renamed_payload["ok"], renamed_payload
|
||||
assert renamed_payload["summary"]["pass_count"] == 5, renamed_payload
|
||||
warnings = [warning for target in renamed_payload["targets"] for warning in target["warnings"]]
|
||||
assert any("source checkout directory differs" in warning for warning in warnings), renamed_payload
|
||||
|
||||
broken = tmp_root / "broken-skill"
|
||||
(broken / "agents").mkdir(parents=True, exist_ok=True)
|
||||
(broken / "reports").mkdir(parents=True, exist_ok=True)
|
||||
|
||||
Reference in New Issue
Block a user