wehub-resource-sync
109 lines
3.7 KiB
Python
109 lines
3.7 KiB
Python
#!/usr/bin/env python3
|
|
"""Run skill_review_checklist_runner on every SKILL.md in the repo + aggregate."""
|
|
|
|
import argparse
|
|
import json
|
|
import os
|
|
import subprocess
|
|
import sys
|
|
|
|
REPO_ROOT = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
|
RUNNER = os.path.join(
|
|
REPO_ROOT,
|
|
"engineering/write-a-skill/skills/write-a-skill/scripts/skill_review_checklist_runner.py"
|
|
)
|
|
|
|
EXCLUDE_PATTERNS = ("node_modules", "/.codex/", "/.gemini/", "/.cursor/", "/.cline/", "/site/", "/.git/", "/templates/", "assets/sample-skill")
|
|
|
|
|
|
def find_skills():
|
|
skills = []
|
|
for root, _, files in os.walk(REPO_ROOT):
|
|
if any(p in root for p in EXCLUDE_PATTERNS):
|
|
continue
|
|
if "SKILL.md" in files:
|
|
skills.append(root)
|
|
return sorted(skills)
|
|
|
|
|
|
def audit_skill(skill_folder):
|
|
"""Run the runner; return parsed JSON result."""
|
|
try:
|
|
out = subprocess.run(
|
|
["python3", RUNNER, skill_folder, "--output", "json"],
|
|
capture_output=True, text=True, timeout=10,
|
|
)
|
|
return json.loads(out.stdout)
|
|
except (subprocess.TimeoutExpired, json.JSONDecodeError, FileNotFoundError) as e:
|
|
return {"error": str(e), "folder": skill_folder, "passed": 0, "total": 6, "overall": "ERROR"}
|
|
|
|
|
|
def main():
|
|
# Parse arguments BEFORE any work: `--help` must return instantly
|
|
# instead of running the full ~30s repo-wide audit.
|
|
parser = argparse.ArgumentParser(
|
|
description="Run the write-a-skill review checklist on every SKILL.md "
|
|
"in the repo and print an aggregate report (~30s on the "
|
|
"full tree). Running with no arguments audits everything.")
|
|
parser.parse_args()
|
|
|
|
skills = find_skills()
|
|
print(f"Auditing {len(skills)} skills...\n", file=sys.stderr)
|
|
|
|
results = []
|
|
for i, folder in enumerate(skills, 1):
|
|
if i % 50 == 0:
|
|
print(f" ... {i}/{len(skills)}", file=sys.stderr)
|
|
results.append(audit_skill(folder))
|
|
|
|
# Aggregate
|
|
pass_count = sum(1 for r in results if r.get("overall") == "PASS")
|
|
warn_count = sum(1 for r in results if r.get("overall") == "WARN")
|
|
fail_count = sum(1 for r in results if r.get("overall") == "FAIL")
|
|
error_count = sum(1 for r in results if r.get("overall") == "ERROR")
|
|
|
|
# Count failures by rule
|
|
rule_failures = {}
|
|
for r in results:
|
|
for check in r.get("checks", []):
|
|
if not check.get("pass"):
|
|
rule = check.get("rule", "unknown")
|
|
rule_failures[rule] = rule_failures.get(rule, 0) + 1
|
|
|
|
# Top-10 worst (most failed checks)
|
|
worst = sorted(
|
|
results,
|
|
key=lambda r: (r.get("passed", 0), -len(r.get("folder", ""))),
|
|
)[:10]
|
|
|
|
print("=" * 72)
|
|
print("REPO-WIDE SKILL AUDIT (write-a-skill review checklist)")
|
|
print("=" * 72)
|
|
print(f"\nTotal SKILL.md audited: {len(results)}")
|
|
print(f" PASS (6/6): {pass_count} ({100*pass_count//max(len(results),1)}%)")
|
|
print(f" WARN (5/6): {warn_count}")
|
|
print(f" FAIL (≤4/6): {fail_count}")
|
|
print(f" ERROR (couldn't parse): {error_count}")
|
|
|
|
print("\n" + "-" * 72)
|
|
print("FAILURES BY RULE")
|
|
print("-" * 72)
|
|
for rule, count in sorted(rule_failures.items(), key=lambda x: -x[1]):
|
|
pct = 100 * count // max(len(results), 1)
|
|
print(f" {count:>4d} ({pct:>3d}%) {rule}")
|
|
|
|
print("\n" + "-" * 72)
|
|
print("TOP-10 WORST OFFENDERS")
|
|
print("-" * 72)
|
|
for w in worst:
|
|
folder = w.get("folder", "?").replace(REPO_ROOT + "/", "")
|
|
passed = w.get("passed", 0)
|
|
failed_rules = [c["rule"] for c in w.get("checks", []) if not c.get("pass")]
|
|
print(f"\n [{passed}/6] {folder}")
|
|
for fr in failed_rules:
|
|
print(f" - {fr}")
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|