feat: add governance and resource boundary quality gates
This commit is contained in:
@@ -1,6 +1,6 @@
|
||||
PYTHON ?= python3
|
||||
|
||||
.PHONY: eval eval-suite results-panel failure-regression-check package-check package-failure-check snapshot-check validate lint test clean
|
||||
.PHONY: eval eval-suite results-panel failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean
|
||||
|
||||
eval:
|
||||
$(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
@@ -29,7 +29,16 @@ validate:
|
||||
lint:
|
||||
$(PYTHON) scripts/lint_skill.py .
|
||||
|
||||
test: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint
|
||||
governance-check:
|
||||
$(PYTHON) scripts/governance_check.py . --require-manifest
|
||||
|
||||
resource-boundary-check:
|
||||
$(PYTHON) scripts/resource_boundary_check.py .
|
||||
|
||||
quality-check:
|
||||
$(PYTHON) tests/verify_quality_checks.py
|
||||
|
||||
test: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check
|
||||
|
||||
clean:
|
||||
rm -rf dist tests/tmp tests/tmp_snapshot
|
||||
|
||||
@@ -19,7 +19,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl
|
||||
|
||||
1. Describe the workflow, prompt set, or repeated task you want to turn into a skill.
|
||||
2. Use `yao-meta-skill` to generate or improve the package in scaffold, production, or library mode.
|
||||
3. Run `context_sizer.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result.
|
||||
3. Run `context_sizer.py`, `resource_boundary_check.py`, `governance_check.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result.
|
||||
|
||||
## 5-Minute Workflow
|
||||
|
||||
@@ -34,6 +34,8 @@ Minimum commands:
|
||||
```bash
|
||||
python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json
|
||||
python3 scripts/context_sizer.py .
|
||||
python3 scripts/resource_boundary_check.py .
|
||||
python3 scripts/governance_check.py . --require-manifest
|
||||
python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip
|
||||
python3 tests/verify_packager_failures.py
|
||||
```
|
||||
@@ -89,6 +91,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa
|
||||
- packaging validation: `openai`, `claude`, and `generic` targets pass contract checks
|
||||
- packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected
|
||||
- failure library regressions: anti-pattern families pass automated checks
|
||||
- governance and resource-boundary checks are part of the default test path
|
||||
|
||||
## What It Does
|
||||
|
||||
@@ -151,6 +154,8 @@ Utility scripts that make the meta-skill operational:
|
||||
- `trigger_eval.py`: evaluates trigger descriptions with semantic intent concepts, explicit exclusions, and near-neighbor prompts
|
||||
- `run_eval_suite.py`: runs train/dev/holdout trigger suites, reports family-level regressions, and fails if aggregate regressions appear
|
||||
- `context_sizer.py`: estimates context weight and warns when the initial load gets too large
|
||||
- `resource_boundary_check.py`: audits whether detail is split across `SKILL.md`, `references/`, `scripts/`, `assets/`, and `evals/` appropriately
|
||||
- `governance_check.py`: validates owner, review cadence, lifecycle stage, and maturity metadata
|
||||
- `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation
|
||||
- `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain
|
||||
|
||||
@@ -175,6 +180,7 @@ Continuous integration entrypoint that runs the full local regression suite on p
|
||||
- `evals/failure-cases.md` captures known weak spots that should remain part of regression checks.
|
||||
- `failures/` captures reusable anti-pattern writeups and machine-runnable failure cases for routing, packaging, and authoring failures.
|
||||
- `tests/verify_packager_failures.py` checks that invalid metadata, invalid YAML, and unsupported targets fail clearly.
|
||||
- Governance metadata and resource-boundary rules now have runnable checks instead of staying as prose only.
|
||||
|
||||
### `templates/`
|
||||
|
||||
@@ -210,6 +216,8 @@ Examples:
|
||||
```bash
|
||||
python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --expectations evals/packaging_expectations.json --zip
|
||||
python3 scripts/context_sizer.py ./yao-meta-skill
|
||||
python3 scripts/resource_boundary_check.py ./yao-meta-skill
|
||||
python3 scripts/governance_check.py ./yao-meta-skill --require-manifest
|
||||
python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt
|
||||
```
|
||||
|
||||
@@ -218,6 +226,7 @@ python3 scripts/trigger_eval.py --description-file evals/improved_description.tx
|
||||
- **Neutral by default**: source files stay vendor-neutral, while adapters are generated only when needed.
|
||||
- **Context efficient**: the project explicitly pushes detail out of the main skill file.
|
||||
- **Evaluation-aware**: trigger and sizing checks are built into the workflow.
|
||||
- **Governed**: important skills can be checked for lifecycle metadata, ownership, and review cadence.
|
||||
- **Reusable**: the output is a package, not just a paragraph of prompt text.
|
||||
- **Portable**: compatibility is handled through packaging rather than duplicating source files for every client.
|
||||
|
||||
@@ -247,10 +256,13 @@ This project is best for:
|
||||
- Failure library: [failures/README.md](failures/README.md)
|
||||
- Failure regression check: [verify_failure_regressions.py](tests/verify_failure_regressions.py)
|
||||
- Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md)
|
||||
- Governance model: [references/governance.md](references/governance.md)
|
||||
- Resource boundary spec: [references/resource-boundaries.md](references/resource-boundaries.md)
|
||||
- Platform capability matrix: [references/platform-capability-matrix.md](references/platform-capability-matrix.md)
|
||||
- Failure fixtures: [tests/fixtures](tests/fixtures)
|
||||
- Adapter snapshots: [tests/snapshots](tests/snapshots)
|
||||
- Evolution example: [examples/evolution-frontend-review/README.md](examples/evolution-frontend-review/README.md)
|
||||
- World-class roadmap: [WORLD_CLASS_PLAN.md](WORLD_CLASS_PLAN.md)
|
||||
|
||||
## License
|
||||
|
||||
|
||||
@@ -73,6 +73,8 @@ Use these when they materially improve quality:
|
||||
- `scripts/trigger_eval.py`
|
||||
- `scripts/context_sizer.py`
|
||||
- `scripts/cross_packager.py`
|
||||
- `scripts/governance_check.py`
|
||||
- `scripts/resource_boundary_check.py`
|
||||
|
||||
## Workflow
|
||||
|
||||
@@ -134,6 +136,8 @@ Use the minimum useful QA:
|
||||
- advanced: trigger evals, benchmark comparisons, revision loop
|
||||
|
||||
For production or library-grade skills, run `scripts/context_sizer.py` before finalizing.
|
||||
Run `scripts/resource_boundary_check.py` when package sprawl or misplaced detail is a risk.
|
||||
Run `scripts/governance_check.py` when the skill is intended to be a maintained shared asset.
|
||||
|
||||
### 6. Package for reuse
|
||||
|
||||
@@ -175,3 +179,5 @@ Unless the user asks otherwise, produce:
|
||||
- [Meta-Skill Rubric](references/design-rubric.md)
|
||||
- [Skill Template](references/skill-template.md)
|
||||
- [Trigger And Eval Playbook](references/eval-playbook.md)
|
||||
- [Governance Model](references/governance.md)
|
||||
- [Resource Boundary Spec](references/resource-boundaries.md)
|
||||
|
||||
@@ -0,0 +1,76 @@
|
||||
# World-Class Plan
|
||||
|
||||
`yao-meta-skill` should not compete by becoming the largest meta-skill. It should compete by becoming the most rigorous lightweight system for creating, evaluating, packaging, and governing reusable agent skills.
|
||||
|
||||
## Positioning
|
||||
|
||||
The intended advantage is:
|
||||
|
||||
- lighter than heavyweight skill-creation systems
|
||||
- more evaluation-aware than purely structural systems
|
||||
- more trustworthy than ambition-heavy skill factories
|
||||
- stronger at lifecycle governance than typical skill creators
|
||||
|
||||
## Scoring Targets
|
||||
|
||||
- Trigger design: `9.4+`
|
||||
- Methodology depth: `9.5+`
|
||||
- Executability: `9.3+`
|
||||
- Evaluation and iteration: `9.5+`
|
||||
- Engineering support: `9.4+`
|
||||
- Cross-platform packaging: `9.2+`
|
||||
- Simplicity: `9.0+`
|
||||
|
||||
## Strategic Pillars
|
||||
|
||||
### 1. Lightweight By Default
|
||||
|
||||
Protect `SKILL.md` from bloat and push detail into the right place.
|
||||
|
||||
### 2. Eval Before Expansion
|
||||
|
||||
Validate routing and boundaries before making the skill bigger.
|
||||
|
||||
### 3. Explicit Boundary Design
|
||||
|
||||
Make scope, exclusions, and near-neighbor behavior legible.
|
||||
|
||||
### 4. Evidence Over Claims
|
||||
|
||||
Expose regressions, family results, failure modes, and change history.
|
||||
|
||||
### 5. Governance As Product
|
||||
|
||||
Treat important skills as maintained assets with owners, cadence, and lifecycle state.
|
||||
|
||||
## Implementation Sequence
|
||||
|
||||
### Phase 1
|
||||
|
||||
- resource boundary spec
|
||||
- governance spec
|
||||
- quality checks wired into validation
|
||||
- visible regression reporting
|
||||
|
||||
### Phase 2
|
||||
|
||||
- thicker benchmark suites
|
||||
- heavier production examples
|
||||
- versioned regression history
|
||||
- maturity scoring and certification ideas
|
||||
|
||||
### Phase 3
|
||||
|
||||
- stronger adapter conformance
|
||||
- broader external benchmark evidence
|
||||
- deeper library governance workflows
|
||||
|
||||
## Acceptance Criteria
|
||||
|
||||
The project is only world-class if it remains:
|
||||
|
||||
- compact at the entrypoint
|
||||
- reproducible in evaluation
|
||||
- explicit in packaging limits
|
||||
- auditable in governance
|
||||
- credible through public evidence
|
||||
@@ -3,6 +3,10 @@
|
||||
"version": "1.0.0",
|
||||
"owner": "Yao Team",
|
||||
"updated_at": "2026-03-31",
|
||||
"status": "active",
|
||||
"maturity_tier": "production",
|
||||
"lifecycle_stage": "production",
|
||||
"review_cadence": "quarterly",
|
||||
"category": "operations",
|
||||
"complexity_tier": "complex",
|
||||
"factory_components": [
|
||||
|
||||
@@ -3,6 +3,9 @@
|
||||
"version": "1.1.0",
|
||||
"owner": "Yao Team",
|
||||
"updated_at": "2026-03-31",
|
||||
"status": "active",
|
||||
"maturity_tier": "governed",
|
||||
"lifecycle_stage": "library",
|
||||
"review_cadence": "quarterly",
|
||||
"target_platforms": [
|
||||
"openai",
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
# Governance Model
|
||||
|
||||
This project treats important skills as governed assets rather than one-shot prompt files.
|
||||
|
||||
## Goals
|
||||
|
||||
- keep shared skills trustworthy over time
|
||||
- make ownership explicit
|
||||
- avoid stale or oversized skill packages
|
||||
- define when a skill should evolve, split, or retire
|
||||
|
||||
## Required Governance Metadata
|
||||
|
||||
For reusable or library-grade skills, `manifest.json` should include:
|
||||
|
||||
- `name`
|
||||
- `version`
|
||||
- `owner`
|
||||
- `updated_at`
|
||||
- `review_cadence`
|
||||
- `status`
|
||||
- `maturity_tier`
|
||||
- `lifecycle_stage`
|
||||
|
||||
## Allowed Values
|
||||
|
||||
### `status`
|
||||
|
||||
- `experimental`
|
||||
- `active`
|
||||
- `deprecated`
|
||||
|
||||
### `maturity_tier`
|
||||
|
||||
- `scaffold`
|
||||
- `production`
|
||||
- `library`
|
||||
- `governed`
|
||||
|
||||
### `lifecycle_stage`
|
||||
|
||||
- `scaffold`
|
||||
- `production`
|
||||
- `library`
|
||||
- `governed`
|
||||
|
||||
### `review_cadence`
|
||||
|
||||
- `monthly`
|
||||
- `quarterly`
|
||||
- `semiannual`
|
||||
- `annual`
|
||||
- `per-release`
|
||||
|
||||
## Governance Rules
|
||||
|
||||
### 1. Owner Required
|
||||
|
||||
Any skill meant for reuse must have a named owner or owning team.
|
||||
|
||||
### 2. Review Cadence Required
|
||||
|
||||
If a skill is shared, it must declare how often it should be reviewed.
|
||||
|
||||
### 3. Maturity Should Match Rigor
|
||||
|
||||
- `scaffold`: lightweight, personal, low-governance
|
||||
- `production`: reusable team skill with validation
|
||||
- `library`: curated shared skill with explicit packaging and evals
|
||||
- `governed`: critical or meta-level skill with regression, maintenance, and review expectations
|
||||
|
||||
### 4. Deprecated Skills Need Explicit Intent
|
||||
|
||||
Deprecated skills should include a deprecation note or replacement reference in adjacent documentation or manifest extensions.
|
||||
|
||||
### 5. Drift Must Be Observable
|
||||
|
||||
Important skills should keep:
|
||||
|
||||
- a regression history
|
||||
- visible evaluation results
|
||||
- known anti-patterns or failure modes
|
||||
|
||||
## Governance Actions
|
||||
|
||||
Use governance review to decide whether a skill should:
|
||||
|
||||
- stay as-is
|
||||
- tighten trigger boundaries
|
||||
- split into sibling skills
|
||||
- move detail into `references/`
|
||||
- move brittle logic into `scripts/`
|
||||
- be deprecated or replaced
|
||||
|
||||
## Why This Matters
|
||||
|
||||
Most skill systems stop at creation. World-class skill systems also manage:
|
||||
|
||||
- ownership
|
||||
- drift
|
||||
- maturity
|
||||
- deprecation
|
||||
- evidence of ongoing quality
|
||||
@@ -0,0 +1,77 @@
|
||||
# Resource Boundary Spec
|
||||
|
||||
This spec defines where information belongs inside a skill package.
|
||||
|
||||
## Principle
|
||||
|
||||
Keep the main skill small enough to route and execute clearly. Move detail out of `SKILL.md` as soon as it stops helping routing or branch selection.
|
||||
|
||||
## Placement Rules
|
||||
|
||||
### Put content in `SKILL.md` when it is:
|
||||
|
||||
- part of the trigger surface
|
||||
- part of the core execution skeleton
|
||||
- part of the output contract
|
||||
- necessary for branch selection or safe defaults
|
||||
|
||||
### Put content in `references/` when it is:
|
||||
|
||||
- domain guidance
|
||||
- long examples
|
||||
- policy material
|
||||
- schemas or templates humans or agents may read on demand
|
||||
|
||||
### Put content in `scripts/` when it is:
|
||||
|
||||
- deterministic
|
||||
- repetitive
|
||||
- brittle if rewritten from prose
|
||||
- easier to validate as code than as instructions
|
||||
|
||||
### Put content in `evals/` when:
|
||||
|
||||
- the skill is reused enough that routing mistakes matter
|
||||
- near-neighbor confusion is likely
|
||||
- quality claims should be reproducible
|
||||
|
||||
### Put content in `assets/` when:
|
||||
|
||||
- the package includes output artifacts, examples, or static files that should not bloat prompt context
|
||||
|
||||
## Anti-Patterns
|
||||
|
||||
Avoid these:
|
||||
|
||||
- storing long policy text directly in `SKILL.md`
|
||||
- adding `references/` with no files that are actually used
|
||||
- adding `scripts/` for logic that is still best expressed in prose
|
||||
- adding `evals/` for one-off or disposable skills
|
||||
- creating every folder by default even when empty
|
||||
|
||||
## Heuristics
|
||||
|
||||
### `SKILL.md`
|
||||
|
||||
- should stay focused
|
||||
- should not become the full knowledge base
|
||||
- should mention any optional directory that materially affects execution
|
||||
|
||||
### `references/`
|
||||
|
||||
- should earn their keep
|
||||
- should usually be named and discoverable from `SKILL.md`
|
||||
|
||||
### `scripts/`
|
||||
|
||||
- should exist only when deterministic logic or formatting logic is real
|
||||
- should be referenced explicitly from `SKILL.md` when required for execution
|
||||
|
||||
### `evals/`
|
||||
|
||||
- should exist when routing or quality claims need to be defended
|
||||
- should be skipped for disposable personal drafts
|
||||
|
||||
## Quality Intent
|
||||
|
||||
The best skill is not the one with the most files. The best skill is the smallest package that still makes the recurring job reliable, reusable, and auditable.
|
||||
@@ -0,0 +1,105 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from datetime import datetime
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ALLOWED_STATUS = {"experimental", "active", "deprecated"}
|
||||
ALLOWED_MATURITY = {"scaffold", "production", "library", "governed"}
|
||||
ALLOWED_REVIEW_CADENCE = {"monthly", "quarterly", "semiannual", "annual", "per-release"}
|
||||
|
||||
|
||||
def load_json(path: Path) -> dict:
|
||||
return json.loads(path.read_text(encoding="utf-8"))
|
||||
|
||||
|
||||
def read_frontmatter(skill_md: Path) -> dict:
|
||||
if not skill_md.exists():
|
||||
return {}
|
||||
text = skill_md.read_text(encoding="utf-8")
|
||||
if not text.startswith("---"):
|
||||
return {}
|
||||
parts = text.split("---", 2)
|
||||
if len(parts) < 3:
|
||||
return {}
|
||||
payload = {}
|
||||
for line in parts[1].splitlines():
|
||||
if ":" not in line:
|
||||
continue
|
||||
key, value = line.split(":", 1)
|
||||
payload[key.strip()] = value.strip().strip("'\"")
|
||||
return payload
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Check skill governance metadata and lifecycle readiness.")
|
||||
parser.add_argument("skill_dir")
|
||||
parser.add_argument("--require-manifest", action="store_true")
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.skill_dir).resolve()
|
||||
manifest_path = root / "manifest.json"
|
||||
skill_md = root / "SKILL.md"
|
||||
failures = []
|
||||
warnings = []
|
||||
details = {"skill_dir": str(root), "manifest_present": manifest_path.exists()}
|
||||
|
||||
frontmatter = read_frontmatter(skill_md)
|
||||
manifest = {}
|
||||
if manifest_path.exists():
|
||||
try:
|
||||
manifest = load_json(manifest_path)
|
||||
except json.JSONDecodeError as exc:
|
||||
failures.append(f"Invalid manifest.json: {exc}")
|
||||
elif args.require_manifest:
|
||||
failures.append("Missing manifest.json")
|
||||
else:
|
||||
warnings.append("No manifest.json; governance metadata is unavailable.")
|
||||
|
||||
if manifest:
|
||||
required = ["name", "version", "owner", "updated_at", "review_cadence", "status", "maturity_tier", "lifecycle_stage"]
|
||||
missing = [field for field in required if not manifest.get(field)]
|
||||
if missing:
|
||||
failures.append(f"Missing manifest fields: {', '.join(missing)}")
|
||||
|
||||
if manifest.get("status") and manifest["status"] not in ALLOWED_STATUS:
|
||||
failures.append(f"Invalid status: {manifest['status']}")
|
||||
if manifest.get("maturity_tier") and manifest["maturity_tier"] not in ALLOWED_MATURITY:
|
||||
failures.append(f"Invalid maturity_tier: {manifest['maturity_tier']}")
|
||||
if manifest.get("lifecycle_stage") and manifest["lifecycle_stage"] not in ALLOWED_MATURITY:
|
||||
failures.append(f"Invalid lifecycle_stage: {manifest['lifecycle_stage']}")
|
||||
if manifest.get("review_cadence") and manifest["review_cadence"] not in ALLOWED_REVIEW_CADENCE:
|
||||
failures.append(f"Invalid review_cadence: {manifest['review_cadence']}")
|
||||
if manifest.get("updated_at"):
|
||||
try:
|
||||
datetime.strptime(manifest["updated_at"], "%Y-%m-%d")
|
||||
except ValueError:
|
||||
failures.append("updated_at must use YYYY-MM-DD")
|
||||
|
||||
if frontmatter.get("name") and manifest.get("name") and frontmatter["name"] != manifest["name"]:
|
||||
failures.append("manifest name does not match SKILL.md frontmatter name")
|
||||
|
||||
if manifest.get("status") == "deprecated" and not manifest.get("deprecation_note"):
|
||||
warnings.append("Deprecated skill should include deprecation_note in manifest.json.")
|
||||
|
||||
report = {
|
||||
"ok": not failures,
|
||||
"failures": failures,
|
||||
"warnings": warnings,
|
||||
"details": {
|
||||
**details,
|
||||
"frontmatter_name": frontmatter.get("name"),
|
||||
"manifest_name": manifest.get("name"),
|
||||
"status": manifest.get("status"),
|
||||
"maturity_tier": manifest.get("maturity_tier"),
|
||||
"review_cadence": manifest.get("review_cadence"),
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if failures:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -20,9 +20,16 @@ def main() -> None:
|
||||
|
||||
skill_md = root / "SKILL.md"
|
||||
if skill_md.exists():
|
||||
lines = skill_md.read_text(encoding="utf-8").splitlines()
|
||||
text = skill_md.read_text(encoding="utf-8")
|
||||
lines = text.splitlines()
|
||||
if len(lines) > 300:
|
||||
warnings.append("SKILL.md is getting long; consider moving detail into references/.")
|
||||
for dirname in ("references", "scripts", "evals", "assets"):
|
||||
path = root / dirname
|
||||
if path.exists() and not any(child.is_file() for child in path.rglob("*")):
|
||||
warnings.append(f"{dirname}/ exists but is empty.")
|
||||
if path.exists() and any(child.is_file() for child in path.rglob("*")) and dirname not in text and dirname.capitalize() not in text:
|
||||
warnings.append(f"{dirname}/ contains files but is not referenced in SKILL.md.")
|
||||
|
||||
print(json.dumps({"ok": not failures, "failures": failures, "warnings": warnings}, ensure_ascii=False, indent=2))
|
||||
if failures:
|
||||
|
||||
@@ -0,0 +1,103 @@
|
||||
#!/usr/bin/env python3
|
||||
import argparse
|
||||
import json
|
||||
from pathlib import Path
|
||||
|
||||
from context_sizer import estimate_tokens, read_text, TEXT_EXTS
|
||||
|
||||
|
||||
OPTIONAL_DIRS = ("references", "scripts", "assets", "evals", "templates")
|
||||
CANONICAL_PATHS = ("SKILL.md", "manifest.json", "agents", "references", "scripts", "assets", "evals", "templates")
|
||||
|
||||
|
||||
def has_files(path: Path) -> bool:
|
||||
return path.exists() and any(child.is_file() for child in path.rglob("*"))
|
||||
|
||||
|
||||
def iter_relevant_files(root: Path) -> list[Path]:
|
||||
files = []
|
||||
for entry in CANONICAL_PATHS:
|
||||
path = root / entry
|
||||
if path.is_file():
|
||||
files.append(path)
|
||||
elif path.is_dir():
|
||||
files.extend(sorted(file for file in path.rglob("*") if file.is_file()))
|
||||
return files
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Check whether a skill package keeps resource boundaries under control.")
|
||||
parser.add_argument("skill_dir")
|
||||
parser.add_argument("--max-initial-tokens", type=int, default=1800)
|
||||
parser.add_argument("--warn-skill-body-tokens", type=int, default=1400)
|
||||
args = parser.parse_args()
|
||||
|
||||
root = Path(args.skill_dir).resolve()
|
||||
skill_md = root / "SKILL.md"
|
||||
failures = []
|
||||
warnings = []
|
||||
|
||||
if not skill_md.exists():
|
||||
failures.append("Missing SKILL.md")
|
||||
report = {"ok": False, "failures": failures, "warnings": warnings}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
raise SystemExit(2)
|
||||
|
||||
files = iter_relevant_files(root)
|
||||
skill_body_tokens = 0
|
||||
other_tokens = 0
|
||||
initial_load_tokens = 0
|
||||
total_text_tokens = 0
|
||||
for path in files:
|
||||
if path.suffix and path.suffix not in TEXT_EXTS and path.name != "SKILL.md":
|
||||
continue
|
||||
text = read_text(path)
|
||||
tokens = estimate_tokens(text)
|
||||
total_text_tokens += tokens
|
||||
rel = path.relative_to(root)
|
||||
if rel.name == "SKILL.md":
|
||||
skill_body_tokens += tokens
|
||||
initial_load_tokens += tokens
|
||||
else:
|
||||
other_tokens += tokens
|
||||
if rel.parts[0] in {"agents"}:
|
||||
initial_load_tokens += tokens
|
||||
|
||||
if initial_load_tokens > args.max_initial_tokens:
|
||||
failures.append(
|
||||
f"Estimated initial-load tokens exceed budget: {initial_load_tokens} > {args.max_initial_tokens}"
|
||||
)
|
||||
if skill_body_tokens > args.warn_skill_body_tokens:
|
||||
warnings.append(f"SKILL.md is getting heavy: {skill_body_tokens} estimated tokens.")
|
||||
|
||||
skill_text = skill_md.read_text(encoding="utf-8")
|
||||
for dirname in OPTIONAL_DIRS:
|
||||
path = root / dirname
|
||||
if path.exists() and not has_files(path):
|
||||
warnings.append(f"{dirname}/ exists but is empty.")
|
||||
continue
|
||||
if has_files(path) and dirname not in skill_text and dirname.capitalize() not in skill_text:
|
||||
warnings.append(f"{dirname}/ contains files but is not referenced explicitly in SKILL.md.")
|
||||
|
||||
if other_tokens and skill_body_tokens / (skill_body_tokens + other_tokens) > 0.75:
|
||||
warnings.append("Most text still lives in SKILL.md; consider moving detail into references/ or scripts/.")
|
||||
|
||||
report = {
|
||||
"ok": not failures,
|
||||
"failures": failures,
|
||||
"warnings": warnings,
|
||||
"stats": {
|
||||
"skill_body_tokens": skill_body_tokens,
|
||||
"other_text_tokens": other_tokens,
|
||||
"estimated_initial_load_tokens": initial_load_tokens,
|
||||
"estimated_total_text_tokens": total_text_tokens,
|
||||
"relevant_file_count": len(files),
|
||||
},
|
||||
}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if failures:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -12,8 +12,10 @@ def main() -> None:
|
||||
|
||||
root = Path(args.skill_dir).resolve()
|
||||
failures = []
|
||||
warnings = []
|
||||
skill_md = root / "SKILL.md"
|
||||
interface = root / "agents" / "interface.yaml"
|
||||
manifest = root / "manifest.json"
|
||||
|
||||
if not skill_md.exists():
|
||||
failures.append("Missing SKILL.md")
|
||||
@@ -38,7 +40,19 @@ def main() -> None:
|
||||
if not meta.get(field):
|
||||
failures.append(f"Missing interface field: {field}")
|
||||
|
||||
print(json.dumps({"ok": not failures, "failures": failures}, ensure_ascii=False, indent=2))
|
||||
if manifest.exists():
|
||||
data = json.loads(manifest.read_text(encoding="utf-8"))
|
||||
for field in ("name", "version", "owner", "updated_at"):
|
||||
if not data.get(field):
|
||||
failures.append(f"Missing manifest field: {field}")
|
||||
if not data.get("review_cadence"):
|
||||
warnings.append("Manifest exists without review_cadence.")
|
||||
if not data.get("status"):
|
||||
warnings.append("Manifest exists without status.")
|
||||
if not data.get("maturity_tier"):
|
||||
warnings.append("Manifest exists without maturity_tier.")
|
||||
|
||||
print(json.dumps({"ok": not failures, "failures": failures, "warnings": warnings}, ensure_ascii=False, indent=2))
|
||||
if failures:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
@@ -0,0 +1,8 @@
|
||||
---
|
||||
name: invalid-governance-skill
|
||||
description: Test fixture for governance validation.
|
||||
---
|
||||
|
||||
# Invalid Governance Skill
|
||||
|
||||
This fixture exists only to verify governance failures.
|
||||
@@ -0,0 +1,8 @@
|
||||
{
|
||||
"name": "wrong-name",
|
||||
"version": "1.0.0",
|
||||
"updated_at": "2026/03/31",
|
||||
"status": "frozen",
|
||||
"maturity_tier": "governed",
|
||||
"lifecycle_stage": "library"
|
||||
}
|
||||
@@ -0,0 +1,74 @@
|
||||
#!/usr/bin/env python3
|
||||
import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
ROOT = Path(__file__).resolve().parent.parent
|
||||
|
||||
|
||||
def run(name: str, cmd: list[str], expect_ok: bool = True, expected_substrings: list[str] | None = None) -> dict:
|
||||
proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True)
|
||||
payload = {}
|
||||
if proc.stdout.strip():
|
||||
try:
|
||||
payload = json.loads(proc.stdout)
|
||||
except json.JSONDecodeError:
|
||||
payload = {"raw_stdout": proc.stdout}
|
||||
|
||||
joined = proc.stdout + "\n" + proc.stderr
|
||||
passed = proc.returncode == 0 if expect_ok else proc.returncode == 2
|
||||
if expected_substrings:
|
||||
passed = passed and all(fragment in joined for fragment in expected_substrings)
|
||||
|
||||
return {
|
||||
"name": name,
|
||||
"passed": passed,
|
||||
"returncode": proc.returncode,
|
||||
"stdout": proc.stdout,
|
||||
"stderr": proc.stderr,
|
||||
"payload": payload,
|
||||
}
|
||||
|
||||
|
||||
def main() -> None:
|
||||
python = sys.executable
|
||||
cases = [
|
||||
run(
|
||||
"root_governance",
|
||||
[python, "scripts/governance_check.py", str(ROOT), "--require-manifest"],
|
||||
),
|
||||
run(
|
||||
"root_resource_boundaries",
|
||||
[python, "scripts/resource_boundary_check.py", str(ROOT)],
|
||||
),
|
||||
run(
|
||||
"complex_example_governance",
|
||||
[python, "scripts/governance_check.py", str(ROOT / "examples" / "complex-release-orchestrator" / "generated-skill"), "--require-manifest"],
|
||||
),
|
||||
run(
|
||||
"complex_example_resource_boundaries",
|
||||
[python, "scripts/resource_boundary_check.py", str(ROOT / "examples" / "complex-release-orchestrator" / "generated-skill")],
|
||||
),
|
||||
run(
|
||||
"invalid_governance_manifest",
|
||||
[python, "scripts/governance_check.py", str(ROOT / "tests" / "fixtures" / "governance_invalid_manifest"), "--require-manifest"],
|
||||
expect_ok=False,
|
||||
expected_substrings=[
|
||||
"Missing manifest fields",
|
||||
"Invalid status",
|
||||
"updated_at must use YYYY-MM-DD",
|
||||
"manifest name does not match",
|
||||
],
|
||||
),
|
||||
]
|
||||
|
||||
report = {"ok": all(case["passed"] for case in cases), "cases": cases}
|
||||
print(json.dumps(report, ensure_ascii=False, indent=2))
|
||||
if not report["ok"]:
|
||||
raise SystemExit(2)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user