From 13319bf92aca7897b61c5a30b7969fa155c261a5 Mon Sep 17 00:00:00 2001 From: yaojingang Date: Tue, 31 Mar 2026 21:59:04 +0800 Subject: [PATCH] feat: add governance and resource boundary quality gates --- Makefile | 13 ++- README.md | 14 ++- SKILL.md | 6 + WORLD_CLASS_PLAN.md | 76 +++++++++++++ .../generated-skill/manifest.json | 4 + manifest.json | 3 + references/governance.md | 103 +++++++++++++++++ references/resource-boundaries.md | 77 +++++++++++++ scripts/governance_check.py | 105 ++++++++++++++++++ scripts/lint_skill.py | 9 +- scripts/resource_boundary_check.py | 103 +++++++++++++++++ scripts/validate_skill.py | 16 ++- .../governance_invalid_manifest/SKILL.md | 8 ++ .../governance_invalid_manifest/manifest.json | 8 ++ tests/verify_quality_checks.py | 74 ++++++++++++ 15 files changed, 614 insertions(+), 5 deletions(-) create mode 100644 WORLD_CLASS_PLAN.md create mode 100644 references/governance.md create mode 100644 references/resource-boundaries.md create mode 100644 scripts/governance_check.py create mode 100644 scripts/resource_boundary_check.py create mode 100644 tests/fixtures/governance_invalid_manifest/SKILL.md create mode 100644 tests/fixtures/governance_invalid_manifest/manifest.json create mode 100644 tests/verify_quality_checks.py diff --git a/Makefile b/Makefile index 98791693..1c4be998 100644 --- a/Makefile +++ b/Makefile @@ -1,6 +1,6 @@ PYTHON ?= python3 -.PHONY: eval eval-suite results-panel failure-regression-check package-check package-failure-check snapshot-check validate lint test clean +.PHONY: eval eval-suite results-panel failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check test clean eval: $(PYTHON) scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt @@ -29,7 +29,16 @@ validate: lint: $(PYTHON) scripts/lint_skill.py . -test: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint +governance-check: + $(PYTHON) scripts/governance_check.py . --require-manifest + +resource-boundary-check: + $(PYTHON) scripts/resource_boundary_check.py . + +quality-check: + $(PYTHON) tests/verify_quality_checks.py + +test: eval eval-suite failure-regression-check package-check package-failure-check snapshot-check validate lint governance-check resource-boundary-check quality-check clean: rm -rf dist tests/tmp tests/tmp_snapshot diff --git a/README.md b/README.md index 0173f374..a47a2f70 100644 --- a/README.md +++ b/README.md @@ -19,7 +19,7 @@ It turns rough workflows, transcripts, prompts, notes, and runbooks into reusabl 1. Describe the workflow, prompt set, or repeated task you want to turn into a skill. 2. Use `yao-meta-skill` to generate or improve the package in scaffold, production, or library mode. -3. Run `context_sizer.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result. +3. Run `context_sizer.py`, `resource_boundary_check.py`, `governance_check.py`, `trigger_eval.py`, and `cross_packager.py` as needed to validate and export the result. ## 5-Minute Workflow @@ -34,6 +34,8 @@ Minimum commands: ```bash python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json python3 scripts/context_sizer.py . +python3 scripts/resource_boundary_check.py . +python3 scripts/governance_check.py . --require-manifest python3 scripts/cross_packager.py . --platform openai --platform claude --platform generic --expectations evals/packaging_expectations.json --zip python3 tests/verify_packager_failures.py ``` @@ -89,6 +91,7 @@ Full reports: [reports/eval_suite.json](reports/eval_suite.json) and [reports/fa - packaging validation: `openai`, `claude`, and `generic` targets pass contract checks - packaging failure fixtures: invalid metadata, invalid YAML, and unsupported targets fail as expected - failure library regressions: anti-pattern families pass automated checks +- governance and resource-boundary checks are part of the default test path ## What It Does @@ -151,6 +154,8 @@ Utility scripts that make the meta-skill operational: - `trigger_eval.py`: evaluates trigger descriptions with semantic intent concepts, explicit exclusions, and near-neighbor prompts - `run_eval_suite.py`: runs train/dev/holdout trigger suites, reports family-level regressions, and fails if aggregate regressions appear - `context_sizer.py`: estimates context weight and warns when the initial load gets too large +- `resource_boundary_check.py`: audits whether detail is split across `SKILL.md`, `references/`, `scripts/`, `assets/`, and `evals/` appropriately +- `governance_check.py`: validates owner, review cadence, lifecycle stage, and maturity metadata - `cross_packager.py`: builds client-specific export artifacts with explicit platform contracts and validation - `init_skill.py`, `lint_skill.py`, `validate_skill.py`, `diff_eval.py`: minimal authoring toolchain @@ -175,6 +180,7 @@ Continuous integration entrypoint that runs the full local regression suite on p - `evals/failure-cases.md` captures known weak spots that should remain part of regression checks. - `failures/` captures reusable anti-pattern writeups and machine-runnable failure cases for routing, packaging, and authoring failures. - `tests/verify_packager_failures.py` checks that invalid metadata, invalid YAML, and unsupported targets fail clearly. +- Governance metadata and resource-boundary rules now have runnable checks instead of staying as prose only. ### `templates/` @@ -210,6 +216,8 @@ Examples: ```bash python3 scripts/cross_packager.py ./yao-meta-skill --platform openai --platform claude --expectations evals/packaging_expectations.json --zip python3 scripts/context_sizer.py ./yao-meta-skill +python3 scripts/resource_boundary_check.py ./yao-meta-skill +python3 scripts/governance_check.py ./yao-meta-skill --require-manifest python3 scripts/trigger_eval.py --description-file evals/improved_description.txt --cases evals/trigger_cases.json --baseline-description-file evals/baseline_description.txt ``` @@ -218,6 +226,7 @@ python3 scripts/trigger_eval.py --description-file evals/improved_description.tx - **Neutral by default**: source files stay vendor-neutral, while adapters are generated only when needed. - **Context efficient**: the project explicitly pushes detail out of the main skill file. - **Evaluation-aware**: trigger and sizing checks are built into the workflow. +- **Governed**: important skills can be checked for lifecycle metadata, ownership, and review cadence. - **Reusable**: the output is a package, not just a paragraph of prompt text. - **Portable**: compatibility is handled through packaging rather than duplicating source files for every client. @@ -247,10 +256,13 @@ This project is best for: - Failure library: [failures/README.md](failures/README.md) - Failure regression check: [verify_failure_regressions.py](tests/verify_failure_regressions.py) - Packaging contracts: [references/packaging-contracts.md](references/packaging-contracts.md) +- Governance model: [references/governance.md](references/governance.md) +- Resource boundary spec: [references/resource-boundaries.md](references/resource-boundaries.md) - Platform capability matrix: [references/platform-capability-matrix.md](references/platform-capability-matrix.md) - Failure fixtures: [tests/fixtures](tests/fixtures) - Adapter snapshots: [tests/snapshots](tests/snapshots) - Evolution example: [examples/evolution-frontend-review/README.md](examples/evolution-frontend-review/README.md) +- World-class roadmap: [WORLD_CLASS_PLAN.md](WORLD_CLASS_PLAN.md) ## License diff --git a/SKILL.md b/SKILL.md index 105fc32f..ca827856 100644 --- a/SKILL.md +++ b/SKILL.md @@ -73,6 +73,8 @@ Use these when they materially improve quality: - `scripts/trigger_eval.py` - `scripts/context_sizer.py` - `scripts/cross_packager.py` +- `scripts/governance_check.py` +- `scripts/resource_boundary_check.py` ## Workflow @@ -134,6 +136,8 @@ Use the minimum useful QA: - advanced: trigger evals, benchmark comparisons, revision loop For production or library-grade skills, run `scripts/context_sizer.py` before finalizing. +Run `scripts/resource_boundary_check.py` when package sprawl or misplaced detail is a risk. +Run `scripts/governance_check.py` when the skill is intended to be a maintained shared asset. ### 6. Package for reuse @@ -175,3 +179,5 @@ Unless the user asks otherwise, produce: - [Meta-Skill Rubric](references/design-rubric.md) - [Skill Template](references/skill-template.md) - [Trigger And Eval Playbook](references/eval-playbook.md) +- [Governance Model](references/governance.md) +- [Resource Boundary Spec](references/resource-boundaries.md) diff --git a/WORLD_CLASS_PLAN.md b/WORLD_CLASS_PLAN.md new file mode 100644 index 00000000..af035119 --- /dev/null +++ b/WORLD_CLASS_PLAN.md @@ -0,0 +1,76 @@ +# World-Class Plan + +`yao-meta-skill` should not compete by becoming the largest meta-skill. It should compete by becoming the most rigorous lightweight system for creating, evaluating, packaging, and governing reusable agent skills. + +## Positioning + +The intended advantage is: + +- lighter than heavyweight skill-creation systems +- more evaluation-aware than purely structural systems +- more trustworthy than ambition-heavy skill factories +- stronger at lifecycle governance than typical skill creators + +## Scoring Targets + +- Trigger design: `9.4+` +- Methodology depth: `9.5+` +- Executability: `9.3+` +- Evaluation and iteration: `9.5+` +- Engineering support: `9.4+` +- Cross-platform packaging: `9.2+` +- Simplicity: `9.0+` + +## Strategic Pillars + +### 1. Lightweight By Default + +Protect `SKILL.md` from bloat and push detail into the right place. + +### 2. Eval Before Expansion + +Validate routing and boundaries before making the skill bigger. + +### 3. Explicit Boundary Design + +Make scope, exclusions, and near-neighbor behavior legible. + +### 4. Evidence Over Claims + +Expose regressions, family results, failure modes, and change history. + +### 5. Governance As Product + +Treat important skills as maintained assets with owners, cadence, and lifecycle state. + +## Implementation Sequence + +### Phase 1 + +- resource boundary spec +- governance spec +- quality checks wired into validation +- visible regression reporting + +### Phase 2 + +- thicker benchmark suites +- heavier production examples +- versioned regression history +- maturity scoring and certification ideas + +### Phase 3 + +- stronger adapter conformance +- broader external benchmark evidence +- deeper library governance workflows + +## Acceptance Criteria + +The project is only world-class if it remains: + +- compact at the entrypoint +- reproducible in evaluation +- explicit in packaging limits +- auditable in governance +- credible through public evidence diff --git a/examples/complex-release-orchestrator/generated-skill/manifest.json b/examples/complex-release-orchestrator/generated-skill/manifest.json index 3cf42072..29871153 100644 --- a/examples/complex-release-orchestrator/generated-skill/manifest.json +++ b/examples/complex-release-orchestrator/generated-skill/manifest.json @@ -3,6 +3,10 @@ "version": "1.0.0", "owner": "Yao Team", "updated_at": "2026-03-31", + "status": "active", + "maturity_tier": "production", + "lifecycle_stage": "production", + "review_cadence": "quarterly", "category": "operations", "complexity_tier": "complex", "factory_components": [ diff --git a/manifest.json b/manifest.json index 528e6906..6a1d80e0 100644 --- a/manifest.json +++ b/manifest.json @@ -3,6 +3,9 @@ "version": "1.1.0", "owner": "Yao Team", "updated_at": "2026-03-31", + "status": "active", + "maturity_tier": "governed", + "lifecycle_stage": "library", "review_cadence": "quarterly", "target_platforms": [ "openai", diff --git a/references/governance.md b/references/governance.md new file mode 100644 index 00000000..c31acc5c --- /dev/null +++ b/references/governance.md @@ -0,0 +1,103 @@ +# Governance Model + +This project treats important skills as governed assets rather than one-shot prompt files. + +## Goals + +- keep shared skills trustworthy over time +- make ownership explicit +- avoid stale or oversized skill packages +- define when a skill should evolve, split, or retire + +## Required Governance Metadata + +For reusable or library-grade skills, `manifest.json` should include: + +- `name` +- `version` +- `owner` +- `updated_at` +- `review_cadence` +- `status` +- `maturity_tier` +- `lifecycle_stage` + +## Allowed Values + +### `status` + +- `experimental` +- `active` +- `deprecated` + +### `maturity_tier` + +- `scaffold` +- `production` +- `library` +- `governed` + +### `lifecycle_stage` + +- `scaffold` +- `production` +- `library` +- `governed` + +### `review_cadence` + +- `monthly` +- `quarterly` +- `semiannual` +- `annual` +- `per-release` + +## Governance Rules + +### 1. Owner Required + +Any skill meant for reuse must have a named owner or owning team. + +### 2. Review Cadence Required + +If a skill is shared, it must declare how often it should be reviewed. + +### 3. Maturity Should Match Rigor + +- `scaffold`: lightweight, personal, low-governance +- `production`: reusable team skill with validation +- `library`: curated shared skill with explicit packaging and evals +- `governed`: critical or meta-level skill with regression, maintenance, and review expectations + +### 4. Deprecated Skills Need Explicit Intent + +Deprecated skills should include a deprecation note or replacement reference in adjacent documentation or manifest extensions. + +### 5. Drift Must Be Observable + +Important skills should keep: + +- a regression history +- visible evaluation results +- known anti-patterns or failure modes + +## Governance Actions + +Use governance review to decide whether a skill should: + +- stay as-is +- tighten trigger boundaries +- split into sibling skills +- move detail into `references/` +- move brittle logic into `scripts/` +- be deprecated or replaced + +## Why This Matters + +Most skill systems stop at creation. World-class skill systems also manage: + +- ownership +- drift +- maturity +- deprecation +- evidence of ongoing quality diff --git a/references/resource-boundaries.md b/references/resource-boundaries.md new file mode 100644 index 00000000..c9db4022 --- /dev/null +++ b/references/resource-boundaries.md @@ -0,0 +1,77 @@ +# Resource Boundary Spec + +This spec defines where information belongs inside a skill package. + +## Principle + +Keep the main skill small enough to route and execute clearly. Move detail out of `SKILL.md` as soon as it stops helping routing or branch selection. + +## Placement Rules + +### Put content in `SKILL.md` when it is: + +- part of the trigger surface +- part of the core execution skeleton +- part of the output contract +- necessary for branch selection or safe defaults + +### Put content in `references/` when it is: + +- domain guidance +- long examples +- policy material +- schemas or templates humans or agents may read on demand + +### Put content in `scripts/` when it is: + +- deterministic +- repetitive +- brittle if rewritten from prose +- easier to validate as code than as instructions + +### Put content in `evals/` when: + +- the skill is reused enough that routing mistakes matter +- near-neighbor confusion is likely +- quality claims should be reproducible + +### Put content in `assets/` when: + +- the package includes output artifacts, examples, or static files that should not bloat prompt context + +## Anti-Patterns + +Avoid these: + +- storing long policy text directly in `SKILL.md` +- adding `references/` with no files that are actually used +- adding `scripts/` for logic that is still best expressed in prose +- adding `evals/` for one-off or disposable skills +- creating every folder by default even when empty + +## Heuristics + +### `SKILL.md` + +- should stay focused +- should not become the full knowledge base +- should mention any optional directory that materially affects execution + +### `references/` + +- should earn their keep +- should usually be named and discoverable from `SKILL.md` + +### `scripts/` + +- should exist only when deterministic logic or formatting logic is real +- should be referenced explicitly from `SKILL.md` when required for execution + +### `evals/` + +- should exist when routing or quality claims need to be defended +- should be skipped for disposable personal drafts + +## Quality Intent + +The best skill is not the one with the most files. The best skill is the smallest package that still makes the recurring job reliable, reusable, and auditable. diff --git a/scripts/governance_check.py b/scripts/governance_check.py new file mode 100644 index 00000000..c3c13b5d --- /dev/null +++ b/scripts/governance_check.py @@ -0,0 +1,105 @@ +#!/usr/bin/env python3 +import argparse +import json +from datetime import datetime +from pathlib import Path + + +ALLOWED_STATUS = {"experimental", "active", "deprecated"} +ALLOWED_MATURITY = {"scaffold", "production", "library", "governed"} +ALLOWED_REVIEW_CADENCE = {"monthly", "quarterly", "semiannual", "annual", "per-release"} + + +def load_json(path: Path) -> dict: + return json.loads(path.read_text(encoding="utf-8")) + + +def read_frontmatter(skill_md: Path) -> dict: + if not skill_md.exists(): + return {} + text = skill_md.read_text(encoding="utf-8") + if not text.startswith("---"): + return {} + parts = text.split("---", 2) + if len(parts) < 3: + return {} + payload = {} + for line in parts[1].splitlines(): + if ":" not in line: + continue + key, value = line.split(":", 1) + payload[key.strip()] = value.strip().strip("'\"") + return payload + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check skill governance metadata and lifecycle readiness.") + parser.add_argument("skill_dir") + parser.add_argument("--require-manifest", action="store_true") + args = parser.parse_args() + + root = Path(args.skill_dir).resolve() + manifest_path = root / "manifest.json" + skill_md = root / "SKILL.md" + failures = [] + warnings = [] + details = {"skill_dir": str(root), "manifest_present": manifest_path.exists()} + + frontmatter = read_frontmatter(skill_md) + manifest = {} + if manifest_path.exists(): + try: + manifest = load_json(manifest_path) + except json.JSONDecodeError as exc: + failures.append(f"Invalid manifest.json: {exc}") + elif args.require_manifest: + failures.append("Missing manifest.json") + else: + warnings.append("No manifest.json; governance metadata is unavailable.") + + if manifest: + required = ["name", "version", "owner", "updated_at", "review_cadence", "status", "maturity_tier", "lifecycle_stage"] + missing = [field for field in required if not manifest.get(field)] + if missing: + failures.append(f"Missing manifest fields: {', '.join(missing)}") + + if manifest.get("status") and manifest["status"] not in ALLOWED_STATUS: + failures.append(f"Invalid status: {manifest['status']}") + if manifest.get("maturity_tier") and manifest["maturity_tier"] not in ALLOWED_MATURITY: + failures.append(f"Invalid maturity_tier: {manifest['maturity_tier']}") + if manifest.get("lifecycle_stage") and manifest["lifecycle_stage"] not in ALLOWED_MATURITY: + failures.append(f"Invalid lifecycle_stage: {manifest['lifecycle_stage']}") + if manifest.get("review_cadence") and manifest["review_cadence"] not in ALLOWED_REVIEW_CADENCE: + failures.append(f"Invalid review_cadence: {manifest['review_cadence']}") + if manifest.get("updated_at"): + try: + datetime.strptime(manifest["updated_at"], "%Y-%m-%d") + except ValueError: + failures.append("updated_at must use YYYY-MM-DD") + + if frontmatter.get("name") and manifest.get("name") and frontmatter["name"] != manifest["name"]: + failures.append("manifest name does not match SKILL.md frontmatter name") + + if manifest.get("status") == "deprecated" and not manifest.get("deprecation_note"): + warnings.append("Deprecated skill should include deprecation_note in manifest.json.") + + report = { + "ok": not failures, + "failures": failures, + "warnings": warnings, + "details": { + **details, + "frontmatter_name": frontmatter.get("name"), + "manifest_name": manifest.get("name"), + "status": manifest.get("status"), + "maturity_tier": manifest.get("maturity_tier"), + "review_cadence": manifest.get("review_cadence"), + }, + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + if failures: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/lint_skill.py b/scripts/lint_skill.py index 13fcc880..779ea14b 100644 --- a/scripts/lint_skill.py +++ b/scripts/lint_skill.py @@ -20,9 +20,16 @@ def main() -> None: skill_md = root / "SKILL.md" if skill_md.exists(): - lines = skill_md.read_text(encoding="utf-8").splitlines() + text = skill_md.read_text(encoding="utf-8") + lines = text.splitlines() if len(lines) > 300: warnings.append("SKILL.md is getting long; consider moving detail into references/.") + for dirname in ("references", "scripts", "evals", "assets"): + path = root / dirname + if path.exists() and not any(child.is_file() for child in path.rglob("*")): + warnings.append(f"{dirname}/ exists but is empty.") + if path.exists() and any(child.is_file() for child in path.rglob("*")) and dirname not in text and dirname.capitalize() not in text: + warnings.append(f"{dirname}/ contains files but is not referenced in SKILL.md.") print(json.dumps({"ok": not failures, "failures": failures, "warnings": warnings}, ensure_ascii=False, indent=2)) if failures: diff --git a/scripts/resource_boundary_check.py b/scripts/resource_boundary_check.py new file mode 100644 index 00000000..b913f36b --- /dev/null +++ b/scripts/resource_boundary_check.py @@ -0,0 +1,103 @@ +#!/usr/bin/env python3 +import argparse +import json +from pathlib import Path + +from context_sizer import estimate_tokens, read_text, TEXT_EXTS + + +OPTIONAL_DIRS = ("references", "scripts", "assets", "evals", "templates") +CANONICAL_PATHS = ("SKILL.md", "manifest.json", "agents", "references", "scripts", "assets", "evals", "templates") + + +def has_files(path: Path) -> bool: + return path.exists() and any(child.is_file() for child in path.rglob("*")) + + +def iter_relevant_files(root: Path) -> list[Path]: + files = [] + for entry in CANONICAL_PATHS: + path = root / entry + if path.is_file(): + files.append(path) + elif path.is_dir(): + files.extend(sorted(file for file in path.rglob("*") if file.is_file())) + return files + + +def main() -> None: + parser = argparse.ArgumentParser(description="Check whether a skill package keeps resource boundaries under control.") + parser.add_argument("skill_dir") + parser.add_argument("--max-initial-tokens", type=int, default=1800) + parser.add_argument("--warn-skill-body-tokens", type=int, default=1400) + args = parser.parse_args() + + root = Path(args.skill_dir).resolve() + skill_md = root / "SKILL.md" + failures = [] + warnings = [] + + if not skill_md.exists(): + failures.append("Missing SKILL.md") + report = {"ok": False, "failures": failures, "warnings": warnings} + print(json.dumps(report, ensure_ascii=False, indent=2)) + raise SystemExit(2) + + files = iter_relevant_files(root) + skill_body_tokens = 0 + other_tokens = 0 + initial_load_tokens = 0 + total_text_tokens = 0 + for path in files: + if path.suffix and path.suffix not in TEXT_EXTS and path.name != "SKILL.md": + continue + text = read_text(path) + tokens = estimate_tokens(text) + total_text_tokens += tokens + rel = path.relative_to(root) + if rel.name == "SKILL.md": + skill_body_tokens += tokens + initial_load_tokens += tokens + else: + other_tokens += tokens + if rel.parts[0] in {"agents"}: + initial_load_tokens += tokens + + if initial_load_tokens > args.max_initial_tokens: + failures.append( + f"Estimated initial-load tokens exceed budget: {initial_load_tokens} > {args.max_initial_tokens}" + ) + if skill_body_tokens > args.warn_skill_body_tokens: + warnings.append(f"SKILL.md is getting heavy: {skill_body_tokens} estimated tokens.") + + skill_text = skill_md.read_text(encoding="utf-8") + for dirname in OPTIONAL_DIRS: + path = root / dirname + if path.exists() and not has_files(path): + warnings.append(f"{dirname}/ exists but is empty.") + continue + if has_files(path) and dirname not in skill_text and dirname.capitalize() not in skill_text: + warnings.append(f"{dirname}/ contains files but is not referenced explicitly in SKILL.md.") + + if other_tokens and skill_body_tokens / (skill_body_tokens + other_tokens) > 0.75: + warnings.append("Most text still lives in SKILL.md; consider moving detail into references/ or scripts/.") + + report = { + "ok": not failures, + "failures": failures, + "warnings": warnings, + "stats": { + "skill_body_tokens": skill_body_tokens, + "other_text_tokens": other_tokens, + "estimated_initial_load_tokens": initial_load_tokens, + "estimated_total_text_tokens": total_text_tokens, + "relevant_file_count": len(files), + }, + } + print(json.dumps(report, ensure_ascii=False, indent=2)) + if failures: + raise SystemExit(2) + + +if __name__ == "__main__": + main() diff --git a/scripts/validate_skill.py b/scripts/validate_skill.py index 6ef9e1c7..835d89bc 100644 --- a/scripts/validate_skill.py +++ b/scripts/validate_skill.py @@ -12,8 +12,10 @@ def main() -> None: root = Path(args.skill_dir).resolve() failures = [] + warnings = [] skill_md = root / "SKILL.md" interface = root / "agents" / "interface.yaml" + manifest = root / "manifest.json" if not skill_md.exists(): failures.append("Missing SKILL.md") @@ -38,7 +40,19 @@ def main() -> None: if not meta.get(field): failures.append(f"Missing interface field: {field}") - print(json.dumps({"ok": not failures, "failures": failures}, ensure_ascii=False, indent=2)) + if manifest.exists(): + data = json.loads(manifest.read_text(encoding="utf-8")) + for field in ("name", "version", "owner", "updated_at"): + if not data.get(field): + failures.append(f"Missing manifest field: {field}") + if not data.get("review_cadence"): + warnings.append("Manifest exists without review_cadence.") + if not data.get("status"): + warnings.append("Manifest exists without status.") + if not data.get("maturity_tier"): + warnings.append("Manifest exists without maturity_tier.") + + print(json.dumps({"ok": not failures, "failures": failures, "warnings": warnings}, ensure_ascii=False, indent=2)) if failures: raise SystemExit(2) diff --git a/tests/fixtures/governance_invalid_manifest/SKILL.md b/tests/fixtures/governance_invalid_manifest/SKILL.md new file mode 100644 index 00000000..7f2c5726 --- /dev/null +++ b/tests/fixtures/governance_invalid_manifest/SKILL.md @@ -0,0 +1,8 @@ +--- +name: invalid-governance-skill +description: Test fixture for governance validation. +--- + +# Invalid Governance Skill + +This fixture exists only to verify governance failures. diff --git a/tests/fixtures/governance_invalid_manifest/manifest.json b/tests/fixtures/governance_invalid_manifest/manifest.json new file mode 100644 index 00000000..fc47924e --- /dev/null +++ b/tests/fixtures/governance_invalid_manifest/manifest.json @@ -0,0 +1,8 @@ +{ + "name": "wrong-name", + "version": "1.0.0", + "updated_at": "2026/03/31", + "status": "frozen", + "maturity_tier": "governed", + "lifecycle_stage": "library" +} diff --git a/tests/verify_quality_checks.py b/tests/verify_quality_checks.py new file mode 100644 index 00000000..7b7dd86c --- /dev/null +++ b/tests/verify_quality_checks.py @@ -0,0 +1,74 @@ +#!/usr/bin/env python3 +import json +import subprocess +import sys +from pathlib import Path + + +ROOT = Path(__file__).resolve().parent.parent + + +def run(name: str, cmd: list[str], expect_ok: bool = True, expected_substrings: list[str] | None = None) -> dict: + proc = subprocess.run(cmd, cwd=ROOT, capture_output=True, text=True) + payload = {} + if proc.stdout.strip(): + try: + payload = json.loads(proc.stdout) + except json.JSONDecodeError: + payload = {"raw_stdout": proc.stdout} + + joined = proc.stdout + "\n" + proc.stderr + passed = proc.returncode == 0 if expect_ok else proc.returncode == 2 + if expected_substrings: + passed = passed and all(fragment in joined for fragment in expected_substrings) + + return { + "name": name, + "passed": passed, + "returncode": proc.returncode, + "stdout": proc.stdout, + "stderr": proc.stderr, + "payload": payload, + } + + +def main() -> None: + python = sys.executable + cases = [ + run( + "root_governance", + [python, "scripts/governance_check.py", str(ROOT), "--require-manifest"], + ), + run( + "root_resource_boundaries", + [python, "scripts/resource_boundary_check.py", str(ROOT)], + ), + run( + "complex_example_governance", + [python, "scripts/governance_check.py", str(ROOT / "examples" / "complex-release-orchestrator" / "generated-skill"), "--require-manifest"], + ), + run( + "complex_example_resource_boundaries", + [python, "scripts/resource_boundary_check.py", str(ROOT / "examples" / "complex-release-orchestrator" / "generated-skill")], + ), + run( + "invalid_governance_manifest", + [python, "scripts/governance_check.py", str(ROOT / "tests" / "fixtures" / "governance_invalid_manifest"), "--require-manifest"], + expect_ok=False, + expected_substrings=[ + "Missing manifest fields", + "Invalid status", + "updated_at must use YYYY-MM-DD", + "manifest name does not match", + ], + ), + ] + + report = {"ok": all(case["passed"] for case in cases), "cases": cases} + print(json.dumps(report, ensure_ascii=False, indent=2)) + if not report["ok"]: + raise SystemExit(2) + + +if __name__ == "__main__": + main()