chore: import upstream snapshot with attribution

This commit is contained in:
wehub-resource-sync
2026-07-13 12:09:03 +08:00
commit 2de7548470
2883 changed files with 374366 additions and 0 deletions
+59
View File
@@ -0,0 +1,59 @@
"""Shared helpers for scripts/ tools.
Currently provides:
- parse_frontmatter: minimal YAML-subset parser for `--- ... ---` blocks in markdown.
No external dependencies. Python 3.10+ (PEP 604 unions in type hints).
"""
from __future__ import annotations
def parse_frontmatter(text: str) -> dict[str, object] | None:
"""Parse a YAML-subset frontmatter block at the top of a markdown string.
Returns the parsed key/value mapping, or None when no frontmatter is present
or the closing `---` is missing.
Supports:
- bare strings: `key: value`
- single-quoted: `key: 'value'`
- double-quoted: `key: "value"`
- lists: `key: [a, b, "c"]`
- inline comment lines beginning with `#`
"""
if not text.startswith("---\n"):
return None
# Closing delimiter: "\n---\n" inside the file, or "\n---" at EOF.
end = text.find("\n---\n", 4)
if end == -1 and text.endswith("\n---"):
end = len(text) - 4
if end == -1:
return None
block = text[4:end].strip("\n")
result: dict[str, object] = {}
for raw in block.splitlines():
# Anchor at column 0: skip comments + indented lines.
if not raw or raw.startswith("#") or raw[0] in (" ", "\t"):
continue
if ":" not in raw:
continue
key, _, value = raw.partition(":")
key = key.strip()
if not key:
continue
value = value.strip()
if value.startswith("[") and value.endswith("]"):
inner = value[1:-1].strip()
result[key] = (
[item.strip().strip("'\"") for item in inner.split(",") if item.strip()]
if inner
else []
)
elif (value.startswith('"') and value.endswith('"')) or (
value.startswith("'") and value.endswith("'")
):
result[key] = value[1:-1]
else:
result[key] = value
return result
+276
View File
@@ -0,0 +1,276 @@
#!/usr/bin/env python3
"""Invariant checks across every lesson directory.
Usage:
python scripts/audit_lessons.py [--phase N] [--json] [--strict]
Exit codes:
0 — clean
1 — issues found
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Iterable
ROOT = Path(__file__).resolve().parent.parent
PHASES_DIR = ROOT / "phases"
LESSON_DIR_RE = re.compile(r"^[0-9]{2}-[a-z0-9][a-z0-9-]*[a-z0-9]$")
PHASE_DIR_RE = re.compile(r"^[0-9]{2}-[a-z0-9][a-z0-9-]*[a-z0-9]$")
MD_LINK_RE = re.compile(r"\[[^\]]*\]\(([^)\s#]+)(?:#[^)]*)?\)")
H1_RE = re.compile(r"^#\s+\S", re.MULTILINE)
CANONICAL_QUIZ_KEYS = {"stage", "question", "options", "correct", "explanation"}
LEGACY_QUIZ_KEYS = {"q", "choices", "answer"}
CODE_IGNORED_NAMES = {"README.md", "AGENTS.md", ".gitkeep", ".DS_Store"}
MIN_DOC_BYTES = 200
MAX_OPTIONS = 6
MIN_OPTIONS = 2
@dataclass
class Issue:
rule: str
lesson: str
file: str
message: str
def to_dict(self) -> dict[str, str]:
return {
"rule": self.rule,
"lesson": self.lesson,
"file": self.file,
"message": self.message,
}
@dataclass
class Audit:
lessons_checked: int = 0
issues: list[Issue] = field(default_factory=list)
def add(self, rule: str, lesson: Path, file: Path | None, message: str) -> None:
rel_lesson = lesson.relative_to(ROOT).as_posix()
rel_file = file.relative_to(ROOT).as_posix() if file else rel_lesson
self.issues.append(Issue(rule, rel_lesson, rel_file, message))
def iter_lesson_dirs(phase_filter: int | None) -> Iterable[Path]:
if not PHASES_DIR.is_dir():
return
for phase in sorted(PHASES_DIR.iterdir()):
if not phase.is_dir():
continue
if not PHASE_DIR_RE.match(phase.name):
continue
if phase_filter is not None:
try:
phase_num = int(phase.name.split("-", 1)[0])
except ValueError:
continue
if phase_num != phase_filter:
continue
for lesson in sorted(phase.iterdir()):
if lesson.is_dir():
yield lesson
def check_lesson_dir_pattern(audit: Audit, lesson: Path) -> bool:
if not LESSON_DIR_RE.match(lesson.name):
audit.add(
"L001",
lesson,
None,
f"lesson dir name does not match NN-slug pattern: {lesson.name!r}",
)
return False
return True
def check_docs_en_md(audit: Audit, lesson: Path) -> str | None:
doc = lesson / "docs" / "en.md"
if not doc.is_file():
audit.add("L002", lesson, doc, "missing docs/en.md")
return None
try:
text = doc.read_text(encoding="utf-8")
except UnicodeDecodeError:
audit.add("L002", lesson, doc, "docs/en.md is not valid UTF-8")
return None
if len(text.encode("utf-8")) < MIN_DOC_BYTES:
audit.add(
"L003",
lesson,
doc,
f"docs/en.md shorter than {MIN_DOC_BYTES} bytes (got {len(text)})",
)
if not H1_RE.search(text):
audit.add("L004", lesson, doc, "docs/en.md missing top-level H1")
return text
def check_code_main(audit: Audit, lesson: Path) -> None:
code_dir = lesson / "code"
if not code_dir.is_dir():
return
for path in code_dir.rglob("*"):
if path.is_file() and path.name not in CODE_IGNORED_NAMES:
return
audit.add("L005", lesson, code_dir, "code/ is empty (no source or config files)")
def check_quiz(audit: Audit, lesson: Path) -> None:
quiz = lesson / "quiz.json"
if not quiz.is_file():
return
try:
raw = quiz.read_text(encoding="utf-8")
data = json.loads(raw)
except (json.JSONDecodeError, UnicodeDecodeError) as exc:
audit.add("L006", lesson, quiz, f"quiz.json not valid JSON: {exc}")
return
if isinstance(data, list):
questions = data
elif isinstance(data, dict):
questions = data.get("questions")
else:
questions = None
if not isinstance(questions, list) or not questions:
audit.add(
"L006",
lesson,
quiz,
"quiz.json must be a non-empty array or a dict with non-empty questions[]",
)
return
for idx, q in enumerate(questions):
if not isinstance(q, dict):
audit.add("L006", lesson, quiz, f"question[{idx}] is not an object")
continue
legacy = LEGACY_QUIZ_KEYS & q.keys()
if legacy:
audit.add(
"L007",
lesson,
quiz,
f"question[{idx}] uses legacy schema keys {sorted(legacy)} "
f"(canonical: {sorted(CANONICAL_QUIZ_KEYS)})",
)
continue
missing = CANONICAL_QUIZ_KEYS - q.keys()
if missing:
audit.add(
"L006",
lesson,
quiz,
f"question[{idx}] missing keys {sorted(missing)}",
)
continue
options = q.get("options")
if not isinstance(options, list) or not (MIN_OPTIONS <= len(options) <= MAX_OPTIONS):
audit.add(
"L008",
lesson,
quiz,
f"question[{idx}] options length must be {MIN_OPTIONS}..{MAX_OPTIONS} "
f"(got {len(options) if isinstance(options, list) else type(options).__name__})",
)
continue
correct = q.get("correct")
if not isinstance(correct, int) or not (0 <= correct < len(options)):
audit.add(
"L009",
lesson,
quiz,
f"question[{idx}] correct={correct!r} not a valid index in options[0..{len(options) - 1}]",
)
def check_internal_links(audit: Audit, lesson: Path, text: str) -> None:
doc = lesson / "docs" / "en.md"
seen: set[str] = set()
for match in MD_LINK_RE.finditer(text):
href = match.group(1).strip()
if href in seen:
continue
seen.add(href)
if href.startswith(("http://", "https://", "mailto:", "data:")):
continue
if href.startswith("/"):
target = ROOT / href.lstrip("/")
else:
target = (doc.parent / href).resolve()
if not target.exists():
audit.add("L010", lesson, doc, f"internal link does not resolve: {href!r}")
def audit_lesson(audit: Audit, lesson: Path) -> None:
audit.lessons_checked += 1
if not check_lesson_dir_pattern(audit, lesson):
return
text = check_docs_en_md(audit, lesson)
check_code_main(audit, lesson)
check_quiz(audit, lesson)
if text is not None:
check_internal_links(audit, lesson, text)
def render_report(audit: Audit) -> str:
by_rule: dict[str, int] = {}
for issue in audit.issues:
by_rule[issue.rule] = by_rule.get(issue.rule, 0) + 1
lines = [
f"audit_lessons.py — {audit.lessons_checked} lesson(s) checked, "
f"{len(audit.issues)} issue(s)",
]
if audit.issues:
lines.append("")
for issue in audit.issues:
lines.append(f" [{issue.rule}] {issue.file}: {issue.message}")
lines.append("")
lines.append("Summary by rule:")
for rule in sorted(by_rule):
lines.append(f" {rule}: {by_rule[rule]}")
return "\n".join(lines)
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("--phase", type=int, default=None, help="restrict to a single phase number")
parser.add_argument("--json", action="store_true", help="emit JSON report on stdout")
parser.add_argument(
"--strict",
action="store_true",
help="treat warnings as errors (currently equivalent to default; reserved)",
)
args = parser.parse_args(argv)
audit = Audit()
for lesson in iter_lesson_dirs(args.phase):
audit_lesson(audit, lesson)
if args.json:
json.dump(
{
"lessons_checked": audit.lessons_checked,
"issues": [issue.to_dict() for issue in audit.issues],
},
sys.stdout,
indent=2,
)
sys.stdout.write("\n")
else:
sys.stdout.write(render_report(audit) + "\n")
return 1 if audit.issues else 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+294
View File
@@ -0,0 +1,294 @@
#!/usr/bin/env python3
"""Build a machine-readable catalog of the entire curriculum.
Requires Python 3.10+ (PEP 604 union types, Path.is_relative_to).
Walks every `phases/NN-slug/MM-slug/` lesson directory on disk and emits a
single JSON document with the truth of what exists in the repo: phases,
lessons, code files, outputs (skills / prompts / agents), and totals.
Usage:
python3 scripts/build_catalog.py # write catalog.json at repo root
python3 scripts/build_catalog.py --out path/to/catalog.json
python3 scripts/build_catalog.py --stdout # write to stdout, do not touch repo
Output shape (schema_version 1):
{
"schema_version": 1,
"totals": {"phases": ..., "lessons": ..., "skills": ..., "prompts": ..., "agents": ..., "code_files": ...},
"phases": [
{
"num": 0,
"slug": "00-setup-and-tooling",
"title": "Setup and Tooling",
"lesson_count": 12,
"lessons": [
{
"num": 1,
"slug": "01-...",
"title": "...", # H1 from docs/en.md
"path": "phases/00-.../01-...",
"has_docs": true,
"has_code": true,
"has_quiz": false,
"has_notebook": false,
"code_files": ["main.py", ...],
"outputs": [
{"type": "skill", "name": "...", "path": "...", "version": "1.0.0", "description": "...", "tags": [...]}
]
}
]
}
]
}
Stdlib only. No dependencies. Reuses frontmatter parsing logic similar to
install_skills.py but inlined to keep the script self-contained.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from pathlib import Path
from typing import Iterable
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _lib import parse_frontmatter as _parse_frontmatter # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
PHASES_DIR = ROOT / "phases"
PHASE_DIR_RE = re.compile(r"^([0-9]{2})-([a-z0-9][a-z0-9-]*)$")
LESSON_DIR_RE = re.compile(r"^([0-9]{2})-([a-z0-9][a-z0-9-]*)$")
H1_RE = re.compile(r"^#\s+(.+?)\s*$", re.MULTILINE)
ARTIFACT_TYPES = ("skill", "prompt", "agent")
CODE_SUFFIXES = {".py", ".ts", ".tsx", ".js", ".mjs", ".rs", ".jl", ".go", ".swift", ".ipynb"}
def slug_to_title(slug: str) -> str:
words = slug.split("-")
fixups = {
"ai": "AI",
"ml": "ML",
"llm": "LLM",
"llms": "LLMs",
"nlp": "NLP",
"rl": "RL",
"mcp": "MCP",
"rag": "RAG",
"api": "API",
"rlhf": "RLHF",
"dpo": "DPO",
"lora": "LoRA",
"cnn": "CNN",
"rnn": "RNN",
"rnns": "RNNs",
"cnns": "CNNs",
"gpt": "GPT",
"tfidf": "TF-IDF",
"pos": "POS",
"ner": "NER",
"asr": "ASR",
"tts": "TTS",
"ios": "iOS",
"lats": "LATS",
"rewoo": "ReWoo",
"htn": "HTN",
"sft": "SFT",
}
return " ".join(fixups.get(w, w.capitalize()) for w in words)
def parse_frontmatter(text: str) -> dict[str, object]:
return _parse_frontmatter(text) or {}
def read_h1(doc_path: Path) -> str | None:
try:
text = doc_path.read_text(encoding="utf-8")
except (FileNotFoundError, UnicodeDecodeError):
return None
match = H1_RE.search(text)
return match.group(1).strip() if match else None
def list_code_files(code_dir: Path) -> list[str]:
if not code_dir.is_dir():
return []
files = []
for path in sorted(code_dir.rglob("*")):
if path.is_file() and path.suffix in CODE_SUFFIXES:
files.append(path.relative_to(code_dir).as_posix())
return files
def parse_artifact(path: Path) -> dict[str, object] | None:
stem = path.stem
artifact_type: str | None = None
for t in ARTIFACT_TYPES:
if stem.startswith(f"{t}-"):
artifact_type = t
break
if artifact_type is None:
return None
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
return None
meta = parse_frontmatter(text)
name = str(meta.get("name", "")).strip() or stem
tags = meta.get("tags", [])
if not isinstance(tags, list):
tags = []
return {
"type": artifact_type,
"name": name,
"path": path.relative_to(ROOT).as_posix(),
"version": str(meta.get("version", "")).strip(),
"description": str(meta.get("description", "")).strip(),
"tags": list(tags),
}
def list_outputs(outputs_dir: Path) -> list[dict[str, object]]:
if not outputs_dir.is_dir():
return []
artifacts: list[dict[str, object]] = []
for path in sorted(outputs_dir.iterdir()):
if path.suffix != ".md" or not path.is_file():
continue
record = parse_artifact(path)
if record is not None:
artifacts.append(record)
return artifacts
def build_lesson_entry(lesson_dir: Path) -> dict[str, object] | None:
match = LESSON_DIR_RE.match(lesson_dir.name)
if not match:
return None
num = int(match.group(1))
slug = match.group(2)
docs_path = lesson_dir / "docs" / "en.md"
code_dir = lesson_dir / "code"
outputs_dir = lesson_dir / "outputs"
notebook_dir = lesson_dir / "notebook"
quiz_path = lesson_dir / "quiz.json"
code_files = list_code_files(code_dir)
outputs = list_outputs(outputs_dir)
has_docs = docs_path.is_file()
if not has_docs and not code_files and not outputs and not quiz_path.is_file():
return None
title = read_h1(docs_path) or slug_to_title(slug)
return {
"num": num,
"slug": lesson_dir.name,
"title": title,
"path": lesson_dir.relative_to(ROOT).as_posix(),
"has_docs": has_docs,
"has_code": code_dir.is_dir(),
"has_quiz": quiz_path.is_file(),
"has_notebook": notebook_dir.is_dir(),
"code_files": code_files,
"outputs": outputs,
}
def iter_phase_dirs() -> Iterable[Path]:
if not PHASES_DIR.is_dir():
return
for path in sorted(PHASES_DIR.iterdir()):
if path.is_dir() and PHASE_DIR_RE.match(path.name):
yield path
def build_phase_entry(phase_dir: Path) -> dict[str, object]:
match = PHASE_DIR_RE.match(phase_dir.name)
assert match is not None
num = int(match.group(1))
slug = match.group(2)
lessons: list[dict[str, object]] = []
for lesson_dir in sorted(phase_dir.iterdir()):
if lesson_dir.is_dir():
entry = build_lesson_entry(lesson_dir)
if entry is not None:
lessons.append(entry)
return {
"num": num,
"slug": phase_dir.name,
"title": slug_to_title(slug),
"lesson_count": len(lessons),
"lessons": lessons,
}
def compute_totals(phases: list[dict[str, object]]) -> dict[str, int]:
totals = {
"phases": len(phases),
"lessons": 0,
"skills": 0,
"prompts": 0,
"agents": 0,
"code_files": 0,
}
for phase in phases:
for lesson in phase["lessons"]:
totals["lessons"] += 1
totals["code_files"] += len(lesson["code_files"])
for artifact in lesson["outputs"]:
key = f"{artifact['type']}s"
totals[key] = totals.get(key, 0) + 1
return totals
def build_catalog() -> dict[str, object]:
phases = [build_phase_entry(p) for p in iter_phase_dirs()]
catalog = {
"schema_version": 1,
"totals": compute_totals(phases),
"phases": phases,
}
return catalog
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--out",
type=Path,
default=ROOT / "catalog.json",
help="output path (default: <repo>/catalog.json)",
)
parser.add_argument(
"--stdout",
action="store_true",
help="write JSON to stdout instead of a file",
)
args = parser.parse_args(argv)
catalog = build_catalog()
payload = json.dumps(catalog, indent=2, ensure_ascii=False) + "\n"
if args.stdout:
sys.stdout.write(payload)
return 0
args.out.parent.mkdir(parents=True, exist_ok=True)
args.out.write_text(payload, encoding="utf-8")
totals = catalog["totals"]
sys.stdout.write(
f"catalog: {args.out.relative_to(ROOT) if args.out.is_relative_to(ROOT) else args.out}\n"
)
sys.stdout.write(
f" phases={totals['phases']} lessons={totals['lessons']} "
f"skills={totals['skills']} prompts={totals['prompts']} "
f"agents={totals['agents']} code_files={totals['code_files']}\n"
)
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+263
View File
@@ -0,0 +1,263 @@
#!/usr/bin/env python3
"""Verify that hardcoded counts in README.md match catalog.json totals.
Requires Python 3.10+. Stdlib only.
catalog.json is filesystem-truth (rebuilt by scripts/build_catalog.py and
checked in CI). The README, however, sprinkles hardcoded counts ("428
lessons", "373 skills, 99 prompts, ...") that drift every time the
curriculum grows or shrinks. This script pins each hardcoded count to a
field in catalog.json's `totals` block and fails when they disagree.
Usage:
python3 scripts/check_readme_counts.py # exit 1 on any drift
python3 scripts/check_readme_counts.py --json # machine-readable report
python3 scripts/check_readme_counts.py --fix # rewrite README to match catalog
The --fix flag is opt-in. CI runs the script without --fix and fails the
build on any mismatch, surfacing the drift in the workflow log.
Patterns are deliberately anchored to README context (badge URLs, alt
attributes, specific prose) so per-phase counts like `<code>22 lessons</code>`
in the Contents table are NOT touched. Each pattern declares its catalog
field and a short human description; mismatches are reported with line
numbers and surrounding text.
"""
from __future__ import annotations
import argparse
import json
import re
import sys
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
CATALOG_PATH = ROOT / "catalog.json"
README_PATH = ROOT / "README.md"
@dataclass(frozen=True)
class CountPattern:
"""A single hardcoded count in README pinned to a catalog totals field."""
regex: re.Pattern[str]
field: str # totals.<field>
description: str
PATTERNS: tuple[CountPattern, ...] = (
CountPattern(
regex=re.compile(r"lessons-(\d+)-3553ff"),
field="lessons",
description="lesson-count badge URL",
),
CountPattern(
regex=re.compile(r'alt="(\d+) lessons"'),
field="lessons",
description="lesson-count badge alt text",
),
CountPattern(
regex=re.compile(r"^> (\d+) lessons\. \d+ phases\.", re.MULTILINE),
field="lessons",
description="hero blockquote lesson count",
),
CountPattern(
regex=re.compile(r"^> \d+ lessons\. (\d+) phases\.", re.MULTILINE),
field="phases",
description="hero blockquote phase count",
),
CountPattern(
regex=re.compile(r"This curriculum is the spine\. (\d+) phases,"),
field="phases",
description="'spine' prose phase count",
),
CountPattern(
regex=re.compile(r"This curriculum is the spine\. \d+ phases, (\d+) lessons,"),
field="lessons",
description="'spine' prose lesson count",
),
CountPattern(
regex=re.compile(r"phases-(\d+)-3553ff"),
field="phases",
description="phase-count badge URL",
),
CountPattern(
regex=re.compile(r'alt="(\d+) phases"'),
field="phases",
description="phase-count badge alt text",
),
CountPattern(
regex=re.compile(r"portfolio of (\d+) artifacts"),
field="lessons",
description="'portfolio of N artifacts' (one artifact per lesson)",
),
CountPattern(
regex=re.compile(r"The repo ships (\d+) skills"),
field="skills",
description="toolkit section skill count",
),
CountPattern(
regex=re.compile(r"The repo ships \d+ skills and (\d+) prompts"),
field="prompts",
description="toolkit section prompt count",
),
CountPattern(
regex=re.compile(r"MIT-licensed, (\d+) lessons\."),
field="lessons",
description="sponsor section lesson count",
),
)
@dataclass
class Mismatch:
pattern: CountPattern
found: int
expected: int
line: int
snippet: str
def load_totals() -> dict[str, int]:
with CATALOG_PATH.open(encoding="utf-8") as fh:
catalog = json.load(fh)
totals = catalog.get("totals")
if not isinstance(totals, dict):
raise SystemExit("catalog.json is missing the 'totals' block")
return totals
def line_for(text: str, offset: int) -> int:
return text.count("\n", 0, offset) + 1
def snippet_for(text: str, offset: int, end: int) -> str:
line_start = text.rfind("\n", 0, offset) + 1
line_end = text.find("\n", end)
if line_end == -1:
line_end = len(text)
return text[line_start:line_end].strip()
def find_mismatches(readme_text: str, totals: dict[str, int]) -> list[Mismatch]:
mismatches: list[Mismatch] = []
for pattern in PATTERNS:
expected = totals.get(pattern.field)
if expected is None:
raise SystemExit(f"catalog.json totals is missing field: {pattern.field}")
matched_any = False
for match in pattern.regex.finditer(readme_text):
matched_any = True
found = int(match.group(1))
if found != expected:
mismatches.append(
Mismatch(
pattern=pattern,
found=found,
expected=expected,
line=line_for(readme_text, match.start()),
snippet=snippet_for(readme_text, match.start(), match.end()),
)
)
if not matched_any:
raise SystemExit(
f"pattern did not match README at all: {pattern.description} "
f"({pattern.regex.pattern!r}). The README structure has changed; "
f"update scripts/check_readme_counts.py."
)
return mismatches
def apply_fixes(readme_text: str, totals: dict[str, int]) -> str:
for pattern in PATTERNS:
expected = totals[pattern.field]
def replace(match: re.Match[str], expected: int = expected) -> str:
whole = match.group(0)
old = match.group(1)
start = match.start(1) - match.start()
return whole[:start] + str(expected) + whole[start + len(old):]
readme_text = pattern.regex.sub(replace, readme_text)
return readme_text
def render_text_report(mismatches: list[Mismatch]) -> str:
if not mismatches:
return "README.md counts match catalog.json totals.\n"
out = [f"README.md drift detected: {len(mismatches)} mismatch(es).\n"]
for m in mismatches:
out.append(
f" README.md:{m.line} {m.pattern.description}\n"
f" expected totals.{m.pattern.field} = {m.expected}, found {m.found}\n"
f" >>> {m.snippet}\n"
)
out.append(
"\nRun `python3 scripts/check_readme_counts.py --fix` to update README.md.\n"
)
return "".join(out)
def render_json_report(mismatches: list[Mismatch], totals: dict[str, int]) -> str:
payload = {
"ok": not mismatches,
"totals": totals,
"mismatches": [
{
"line": m.line,
"field": m.pattern.field,
"description": m.pattern.description,
"expected": m.expected,
"found": m.found,
"snippet": m.snippet,
}
for m in mismatches
],
}
return json.dumps(payload, indent=2) + "\n"
def main(argv: list[str] | None = None) -> int:
parser = argparse.ArgumentParser(description=__doc__.split("\n\n")[0])
parser.add_argument("--json", action="store_true", help="emit JSON report on stdout")
parser.add_argument(
"--fix",
action="store_true",
help="rewrite README.md so hardcoded counts match catalog.json",
)
args = parser.parse_args(argv)
totals = load_totals()
readme_text = README_PATH.read_text(encoding="utf-8")
if args.fix:
initial_mismatches = find_mismatches(readme_text, totals)
if not initial_mismatches:
if args.json:
sys.stdout.write(render_json_report([], totals))
else:
sys.stdout.write("README.md already matches catalog.json totals.\n")
return 0
new_text = apply_fixes(readme_text, totals)
README_PATH.write_text(new_text, encoding="utf-8")
remaining = find_mismatches(new_text, totals)
if args.json:
sys.stdout.write(render_json_report(remaining, totals))
else:
sys.stdout.write("README.md updated to match catalog.json totals.\n")
if remaining:
sys.stdout.write(render_text_report(remaining))
return 1 if remaining else 0
mismatches = find_mismatches(readme_text, totals)
if args.json:
sys.stdout.write(render_json_report(mismatches, totals))
else:
sys.stdout.write(render_text_report(mismatches))
return 1 if mismatches else 0
if __name__ == "__main__":
sys.exit(main())
+291
View File
@@ -0,0 +1,291 @@
#!/usr/bin/env python3
"""Install course outputs (skills / prompts / agents) into a target directory.
Walks every `phases/**/outputs/{skill,prompt,agent}-*.md` artifact across the
curriculum, parses YAML frontmatter, filters by type / phase / tag, and copies
the matching files into a target directory using one of three layouts.
Usage:
python3 scripts/install_skills.py <target_dir> [options]
Options:
--type {skill,prompt,agent,all} default: skill
--phase N filter to a single phase number
--tag TAG filter to outputs whose tags include TAG
--layout {flat,by-phase,skills} default: skills
flat <target>/<name>.md
by-phase <target>/phase-NN/<name>.md
skills <target>/<name>/SKILL.md
--dry-run preview without writing
--force overwrite existing files
--json write manifest.json only; do not print steps
Always writes <target>/manifest.json with the full inventory (name, type, phase,
lesson, source path, target path, tags, version).
"""
from __future__ import annotations
import argparse
import json
import shutil
import sys
from dataclasses import dataclass, field
from pathlib import Path
from typing import Iterable
sys.path.insert(0, str(Path(__file__).resolve().parent))
from _lib import parse_frontmatter # noqa: E402
ROOT = Path(__file__).resolve().parent.parent
PHASES_DIR = ROOT / "phases"
VALID_TYPES = ("skill", "prompt", "agent")
LAYOUTS = ("flat", "by-phase", "skills")
@dataclass
class Artifact:
type: str
name: str
phase: int | None
lesson: int | None
version: str
description: str
tags: list[str]
source: Path
def to_dict(self, target: Path | None = None) -> dict:
out: dict[str, object] = {
"type": self.type,
"name": self.name,
"phase": self.phase,
"lesson": self.lesson,
"version": self.version,
"description": self.description,
"tags": self.tags,
"source": self.source.relative_to(ROOT).as_posix(),
}
if target is not None:
out["target"] = target.as_posix()
return out
def derive_phase_lesson(path: Path) -> tuple[int | None, int | None]:
parts = path.parts
phase_num: int | None = None
lesson_num: int | None = None
for part in parts:
if part.startswith(("0", "1", "2")) and "-" in part:
head = part.split("-", 1)[0]
if head.isdigit():
num = int(head)
if phase_num is None:
phase_num = num
elif lesson_num is None:
lesson_num = num
break
return phase_num, lesson_num
def discover_artifacts() -> Iterable[Artifact]:
if not PHASES_DIR.is_dir():
return
for output_dir in sorted(PHASES_DIR.glob("*/[0-9][0-9]-*/outputs")):
for path in sorted(output_dir.iterdir()):
if path.suffix != ".md" or not path.is_file():
continue
stem = path.stem
artifact_type: str | None = None
for t in VALID_TYPES:
if stem.startswith(f"{t}-"):
artifact_type = t
break
if artifact_type is None:
continue
try:
text = path.read_text(encoding="utf-8")
except UnicodeDecodeError:
continue
meta = parse_frontmatter(text) or {}
default_phase, default_lesson = derive_phase_lesson(path)
phase_raw = meta.get("phase", default_phase)
lesson_raw = meta.get("lesson", default_lesson)
try:
phase = int(phase_raw) if phase_raw is not None else None
except (TypeError, ValueError):
phase = default_phase
try:
lesson = int(lesson_raw) if lesson_raw is not None else None
except (TypeError, ValueError):
lesson = default_lesson
name = str(meta.get("name", "")).strip() or stem
description = str(meta.get("description", "")).strip()
version = str(meta.get("version", "")).strip()
tags_raw = meta.get("tags", [])
tags = list(tags_raw) if isinstance(tags_raw, list) else []
yield Artifact(
type=artifact_type,
name=name,
phase=phase,
lesson=lesson,
version=version,
description=description,
tags=tags,
source=path,
)
def filter_artifacts(
artifacts: Iterable[Artifact],
type_filter: str,
phase_filter: int | None,
tag_filter: str | None,
) -> list[Artifact]:
out: list[Artifact] = []
for a in artifacts:
if type_filter != "all" and a.type != type_filter:
continue
if phase_filter is not None and a.phase != phase_filter:
continue
if tag_filter is not None and tag_filter not in a.tags:
continue
out.append(a)
return out
def target_path(artifact: Artifact, target_root: Path, layout: str) -> Path:
if layout == "flat":
return target_root / f"{artifact.name}.md"
if layout == "by-phase":
phase_dir = f"phase-{artifact.phase:02d}" if artifact.phase is not None else "phase-unknown"
return target_root / phase_dir / f"{artifact.name}.md"
if layout == "skills":
return target_root / artifact.name / "SKILL.md"
raise ValueError(f"unknown layout: {layout}")
@dataclass
class Plan:
actions: list[tuple[Artifact, Path]] = field(default_factory=list)
collisions: list[Path] = field(default_factory=list)
def build_plan(
artifacts: list[Artifact], target_root: Path, layout: str, force: bool
) -> Plan:
plan = Plan()
seen_targets: dict[Path, Artifact] = {}
for a in artifacts:
dest = target_path(a, target_root, layout)
if dest in seen_targets:
sys.stderr.write(
f"warn: target collision between {seen_targets[dest].source} "
f"and {a.source} (both map to {dest}); skipping latter\n"
)
continue
seen_targets[dest] = a
if dest.exists() and not force:
plan.collisions.append(dest)
plan.actions.append((a, dest))
return plan
def apply_plan(plan: Plan) -> None:
for artifact, dest in plan.actions:
dest.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(artifact.source, dest)
def write_manifest(target_root: Path, artifacts: list[Artifact], layout: str) -> Path:
manifest_path = target_root / "manifest.json"
target_root.mkdir(parents=True, exist_ok=True)
by_type: dict[str, int] = {}
by_phase: dict[str, int] = {}
entries = []
for a in artifacts:
dest_rel = target_path(a, target_root, layout).relative_to(target_root)
entries.append(a.to_dict(target=dest_rel))
by_type[a.type] = by_type.get(a.type, 0) + 1
key = f"phase-{a.phase:02d}" if a.phase is not None else "phase-unknown"
by_phase[key] = by_phase.get(key, 0) + 1
manifest = {
"schema_version": 1,
"layout": layout,
"totals": {
"artifacts": len(entries),
"by_type": dict(sorted(by_type.items())),
"by_phase": dict(sorted(by_phase.items())),
},
"artifacts": entries,
}
manifest_path.write_text(
json.dumps(manifest, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
return manifest_path
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("target_dir", type=Path)
parser.add_argument("--type", choices=(*VALID_TYPES, "all"), default="skill")
parser.add_argument("--phase", type=int, default=None)
parser.add_argument("--tag", default=None)
parser.add_argument("--layout", choices=LAYOUTS, default="skills")
parser.add_argument("--dry-run", action="store_true")
parser.add_argument("--force", action="store_true")
parser.add_argument(
"--json",
action="store_true",
help="suppress human-readable output (manifest.json still written unless --dry-run)",
)
args = parser.parse_args(argv)
artifacts = list(discover_artifacts())
selected = filter_artifacts(artifacts, args.type, args.phase, args.tag)
if not selected:
sys.stderr.write("no artifacts matched the given filters\n")
return 1
plan = build_plan(selected, args.target_dir, args.layout, args.force)
if plan.collisions and not args.force:
sys.stderr.write(
f"error: {len(plan.collisions)} target file(s) already exist. "
f"Pass --force to overwrite.\n"
)
if not args.json:
for c in plan.collisions[:10]:
sys.stderr.write(f" {c}\n")
if len(plan.collisions) > 10:
sys.stderr.write(f" ... and {len(plan.collisions) - 10} more\n")
return 1
if args.dry_run:
if not args.json:
sys.stdout.write(
f"dry run: {len(plan.actions)} artifact(s) -> {args.target_dir} "
f"(layout={args.layout})\n"
)
for artifact, _dest in plan.actions[:20]:
sys.stdout.write(
f" [{artifact.type}] {artifact.name} "
f"<- {artifact.source.relative_to(ROOT)}\n"
)
if len(plan.actions) > 20:
sys.stdout.write(f" ... and {len(plan.actions) - 20} more\n")
return 0
apply_plan(plan)
manifest_path = write_manifest(args.target_dir, selected, args.layout)
if not args.json:
sys.stdout.write(
f"installed {len(plan.actions)} artifact(s) into {args.target_dir} "
f"(layout={args.layout})\n"
)
sys.stdout.write(f"manifest: {manifest_path}\n")
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+241
View File
@@ -0,0 +1,241 @@
#!/usr/bin/env python3
"""Smoke-check every lesson's Python code.
By default this script byte-compiles every `.py` file under
`phases/**/[0-9][0-9]-*/code/` using `py_compile`. It does NOT execute the
code — that would need API keys and heavy ML dependencies the curriculum
does not pin. Syntax-only is enough to catch the regressions contributors
introduce most often (bad indentation, broken f-strings, stray edits).
Opt in to real execution with `--execute`. Each file runs with a 10-second
timeout. Lessons whose entry file starts with a `# requires: pkg1, pkg2`
comment listing imports outside the standard library are skipped with a
"needs <deps>" reason so heavy lessons (torch, anthropic, etc.) do not blow
up the run.
Usage:
python3 scripts/lesson_run.py # syntax check, full curriculum
python3 scripts/lesson_run.py --phase 14 # one phase only
python3 scripts/lesson_run.py --strict # exit 1 on any failure
python3 scripts/lesson_run.py --json # JSON report on stdout
python3 scripts/lesson_run.py --execute # actually run each lesson
Exit codes:
0 — clean, or non-strict run with failures reported
1 — `--strict` and at least one lesson failed
Stdlib only. Python 3.10+ syntax (PEP 604 unions).
"""
from __future__ import annotations
import argparse
import json
import py_compile
import re
import subprocess
import sys
from dataclasses import asdict, dataclass, field
from pathlib import Path
from typing import Iterable
ROOT = Path(__file__).resolve().parent.parent
PHASES_DIR = ROOT / "phases"
PHASE_DIR_RE = re.compile(r"^([0-9]{2})-[a-z0-9][a-z0-9-]*$")
LESSON_DIR_RE = re.compile(r"^([0-9]{2})-[a-z0-9][a-z0-9-]*$")
REQUIRES_RE = re.compile(r"^\s*#\s*requires:\s*(.+?)\s*$")
EXECUTE_TIMEOUT_SEC = 10
@dataclass
class LessonResult:
lesson: str
files: list[str] = field(default_factory=list)
status: str = "passed" # passed | failed | skipped
reason: str = ""
def to_dict(self) -> dict[str, object]:
return asdict(self)
def iter_lesson_dirs(phase_filter: int | None) -> Iterable[Path]:
if not PHASES_DIR.is_dir():
return
for phase in sorted(PHASES_DIR.iterdir()):
if not phase.is_dir():
continue
m = PHASE_DIR_RE.match(phase.name)
if not m:
continue
if phase_filter is not None and int(m.group(1)) != phase_filter:
continue
for lesson in sorted(phase.iterdir()):
if lesson.is_dir() and LESSON_DIR_RE.match(lesson.name):
yield lesson
def list_python_files(code_dir: Path) -> list[Path]:
if not code_dir.is_dir():
return []
return sorted(p for p in code_dir.rglob("*.py") if p.is_file())
def pick_entry_file(py_files: list[Path]) -> Path | None:
for path in py_files:
if path.name.startswith("main."):
return path
return py_files[0] if py_files else None
def read_requires(path: Path) -> list[str]:
try:
text = path.read_text(encoding="utf-8")
except (UnicodeDecodeError, OSError):
return []
for line in text.splitlines():
stripped = line.strip()
if not stripped:
continue
if not stripped.startswith("#"):
break
match = REQUIRES_RE.match(line)
if match:
deps = [d.strip() for d in match.group(1).split(",")]
return [d for d in deps if d]
return []
def syntax_check(py_files: list[Path]) -> tuple[bool, str]:
for path in py_files:
try:
py_compile.compile(str(path), doraise=True)
except py_compile.PyCompileError as exc:
return False, f"{path.relative_to(ROOT).as_posix()}: {exc.msg.strip()}"
return True, ""
def execute_lesson(entry: Path) -> tuple[bool, str]:
try:
proc = subprocess.run(
[sys.executable, str(entry)],
cwd=str(entry.parent),
stdout=subprocess.DEVNULL,
stderr=subprocess.PIPE,
timeout=EXECUTE_TIMEOUT_SEC,
check=False,
text=True,
)
except subprocess.TimeoutExpired:
return False, f"timeout after {EXECUTE_TIMEOUT_SEC}s"
except OSError as exc:
return False, f"failed to launch interpreter: {exc}"
if proc.returncode == 0:
return True, ""
stderr = (proc.stderr or "").strip()
last_line = stderr.splitlines()[-1] if stderr else f"exit {proc.returncode}"
return False, f"exit {proc.returncode}: {last_line}"
def check_lesson(lesson: Path, execute: bool) -> LessonResult:
rel = lesson.relative_to(ROOT).as_posix()
code_dir = lesson / "code"
py_files = list_python_files(code_dir)
result = LessonResult(
lesson=rel,
files=[p.relative_to(ROOT).as_posix() for p in py_files],
)
if not py_files:
result.status = "skipped"
result.reason = "no python files"
return result
ok, msg = syntax_check(py_files)
if not ok:
result.status = "failed"
result.reason = msg
return result
if execute:
entry = pick_entry_file(py_files)
if entry is None:
result.status = "skipped"
result.reason = "no entry file"
return result
deps = read_requires(entry)
if deps:
result.status = "skipped"
result.reason = f"needs {', '.join(deps)}"
return result
ok, msg = execute_lesson(entry)
if not ok:
result.status = "failed"
result.reason = msg
return result
def render_report(results: list[LessonResult], execute: bool) -> str:
passed = [r for r in results if r.status == "passed"]
failed = [r for r in results if r.status == "failed"]
skipped = [r for r in results if r.status == "skipped"]
mode = "execute" if execute else "syntax"
total_files = sum(len(r.files) for r in results)
lines = [
f"lesson_run.py ({mode}) — {len(results)} lesson(s), "
f"{total_files} python file(s): "
f"passed={len(passed)} failed={len(failed)} skipped={len(skipped)}",
]
if failed:
lines.append("")
lines.append("Failures:")
for r in failed:
lines.append(f" [FAIL] {r.lesson}: {r.reason}")
if skipped and execute:
lines.append("")
lines.append("Skipped:")
for r in skipped:
lines.append(f" [SKIP] {r.lesson}: {r.reason}")
return "\n".join(lines)
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument(
"--phase", type=int, default=None, help="restrict to a single phase number"
)
parser.add_argument(
"--json", action="store_true", help="emit JSON report on stdout"
)
parser.add_argument(
"--strict", action="store_true", help="exit 1 if any lesson fails"
)
parser.add_argument(
"--execute",
action="store_true",
help=f"run each lesson's entry file with a {EXECUTE_TIMEOUT_SEC}s timeout",
)
args = parser.parse_args(argv)
results = [check_lesson(lesson, args.execute) for lesson in iter_lesson_dirs(args.phase)]
failed = [r for r in results if r.status == "failed"]
if args.json:
payload = {
"mode": "execute" if args.execute else "syntax",
"checked": len(results),
"passed": [r.to_dict() for r in results if r.status == "passed"],
"failed": [r.to_dict() for r in results if r.status == "failed"],
"skipped": [r.to_dict() for r in results if r.status == "skipped"],
}
json.dump(payload, sys.stdout, indent=2)
sys.stdout.write("\n")
else:
sys.stdout.write(render_report(results, args.execute) + "\n")
return 1 if (args.strict and failed) else 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))
+442
View File
@@ -0,0 +1,442 @@
#!/usr/bin/env python3
"""Validate external HTTP/HTTPS links in every markdown doc.
Requires Python 3.10+ (PEP 604 union types).
Walks every `*.md` file under the repo (excluding `.git/`, `node_modules/`,
`outputs/`), extracts `https?://` URLs from markdown link syntax and bare URLs,
deduplicates, and validates each unique URL by HEAD request (falling back to
GET on 405/501). Results are cached for 7 days at `.link-cache.json` (repo
root, gitignored) so re-runs do not hammer external services.
Stdlib only. No `requests`, no `httpx`.
Usage:
python3 scripts/link_check.py # full check, group by file
python3 scripts/link_check.py --phase 14 # one phase
python3 scripts/link_check.py --path README.md # one file
python3 scripts/link_check.py --path phases/14-... # one directory
python3 scripts/link_check.py --strict # exit 1 on any broken link
python3 scripts/link_check.py --json # machine-readable
python3 scripts/link_check.py --cache 0 # bypass cache for this run
python3 scripts/link_check.py --timeout 15 # per-request timeout (sec)
python3 scripts/link_check.py --concurrency 16 # worker threads
Companion to `scripts/audit_lessons.py` (rule L010 validates *internal* links);
this script handles the external HTTP/HTTPS surface.
"""
from __future__ import annotations
import argparse
import json
import os
import re
import socket
import ssl
import sys
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from dataclasses import dataclass, field
from pathlib import Path
from typing import Iterable
from urllib import error as urlerror
from urllib import request as urlrequest
from urllib.parse import urlparse
ROOT = Path(__file__).resolve().parent.parent
CACHE_PATH = ROOT / ".link-cache.json"
CACHE_SCHEMA_VERSION = 1
USER_AGENT = (
"ai-engineering-from-scratch link-check/1.0 "
"(+https://aiengineeringfromscratch.com)"
)
DEFAULT_TIMEOUT = 10
DEFAULT_CONCURRENCY = 8
DEFAULT_CACHE_DAYS = 7
DEFAULT_SKIP_DOMAINS = (
"twitter.com",
"x.com",
"linkedin.com",
"instagram.com",
"medium.com",
)
EXCLUDE_DIRS = {".git", "node_modules", "outputs"}
MD_LINK_RE = re.compile(r"\[[^\]]*\]\((<?)(https?://[^\s)>]+)>?\)")
BARE_URL_RE = re.compile(r"(?<![\w(\[=\"'])(https?://[^\s)\]<>\"'`]+)")
TRAILING_PUNCT = ".,;:!?)\"'>"
@dataclass
class UrlOccurrence:
file: str
line: int
@dataclass
class CheckResult:
url: str
status: str
http_status: int | None
error: str | None
cached: bool = False
@property
def ok(self) -> bool:
return self.status == "ok"
@dataclass
class Report:
checked_files: int = 0
unique_urls: int = 0
requested: int = 0
cached_hits: int = 0
skipped: list[str] = field(default_factory=list)
failed: list[dict[str, object]] = field(default_factory=list)
by_file: dict[str, list[dict[str, object]]] = field(default_factory=dict)
def to_dict(self) -> dict[str, object]:
return {
"schema_version": 1,
"checked_files": self.checked_files,
"unique_urls": self.unique_urls,
"requested": self.requested,
"cached_hits": self.cached_hits,
"skipped_count": len(self.skipped),
"failed_count": len(self.failed),
"skipped": sorted(set(self.skipped)),
"failed": self.failed,
"by_file": self.by_file,
}
def iter_markdown_files(
root: Path, phase: int | None, path: Path | None
) -> Iterable[Path]:
if path is not None:
path = path.resolve()
if path.is_file():
if path.suffix == ".md":
yield path
return
roots = [path]
elif phase is not None:
phase_prefix = f"{phase:02d}-"
phases_dir = root / "phases"
if not phases_dir.is_dir():
return
matches = [p for p in phases_dir.iterdir() if p.is_dir() and p.name.startswith(phase_prefix)]
if not matches:
return
roots = matches
else:
roots = [root]
for r in roots:
if r.is_file():
if r.suffix == ".md":
yield r
continue
for dirpath, dirnames, filenames in os.walk(r):
dirnames[:] = [d for d in dirnames if d not in EXCLUDE_DIRS]
for name in filenames:
if name.endswith(".md"):
yield Path(dirpath) / name
def strip_trailing_punct(url: str) -> str:
while url and url[-1] in TRAILING_PUNCT:
url = url[:-1]
return url
def extract_urls(text: str) -> list[tuple[str, int]]:
"""Return list of (url, line_number) tuples preserving order."""
out: list[tuple[str, int]] = []
seen_per_line: set[tuple[int, str]] = set()
for lineno, line in enumerate(text.splitlines(), start=1):
for m in MD_LINK_RE.finditer(line):
url = strip_trailing_punct(m.group(2))
key = (lineno, url)
if key in seen_per_line:
continue
seen_per_line.add(key)
out.append((url, lineno))
masked = MD_LINK_RE.sub(" ", line)
for m in BARE_URL_RE.finditer(masked):
url = strip_trailing_punct(m.group(1))
key = (lineno, url)
if key in seen_per_line:
continue
seen_per_line.add(key)
out.append((url, lineno))
return out
def load_cache() -> dict[str, dict[str, object]]:
if not CACHE_PATH.is_file():
return {}
try:
raw = json.loads(CACHE_PATH.read_text(encoding="utf-8"))
except (OSError, json.JSONDecodeError):
return {}
if not isinstance(raw, dict):
return {}
if raw.get("schema_version") != CACHE_SCHEMA_VERSION:
return {}
entries = raw.get("entries")
return entries if isinstance(entries, dict) else {}
def save_cache(entries: dict[str, dict[str, object]]) -> None:
payload = {"schema_version": CACHE_SCHEMA_VERSION, "entries": entries}
try:
CACHE_PATH.write_text(json.dumps(payload, indent=2, sort_keys=True), encoding="utf-8")
except OSError as exc:
print(f"warning: could not write {CACHE_PATH.name}: {exc}", file=sys.stderr)
def cache_is_fresh(entry: dict[str, object], cache_days: int) -> bool:
if cache_days <= 0:
return False
checked_at = entry.get("checked_at")
if not isinstance(checked_at, (int, float)):
return False
age = time.time() - float(checked_at)
return age < cache_days * 86400
def domain_of(url: str) -> str:
try:
netloc = urlparse(url).netloc.lower()
except ValueError:
return ""
if netloc.startswith("www."):
netloc = netloc[4:]
if ":" in netloc:
netloc = netloc.split(":", 1)[0]
return netloc
def should_skip(url: str, skip_domains: set[str]) -> bool:
domain = domain_of(url)
if not domain:
return False
for sd in skip_domains:
if domain == sd or domain.endswith("." + sd):
return True
return False
def _request(url: str, method: str, timeout: int) -> tuple[int | None, str | None]:
req = urlrequest.Request(
url,
method=method,
headers={
"User-Agent": USER_AGENT,
"Accept": "*/*",
"Accept-Language": "en-US,en;q=0.9",
},
)
ctx = ssl.create_default_context()
try:
with urlrequest.urlopen(req, timeout=timeout, context=ctx) as resp:
return resp.status, None
except urlerror.HTTPError as exc:
return exc.code, f"http {exc.code}"
except urlerror.URLError as exc:
reason = getattr(exc, "reason", exc)
return None, f"url-error: {reason}"
except socket.timeout:
return None, "timeout"
except (TimeoutError, ConnectionError) as exc:
return None, f"conn-error: {exc}"
except ssl.SSLError as exc:
return None, f"ssl-error: {exc}"
except Exception as exc:
return None, f"error: {exc.__class__.__name__}: {exc}"
def check_url(url: str, timeout: int) -> CheckResult:
status_code, err = _request(url, "HEAD", timeout)
if status_code in (405, 501) or (status_code is None and err and "http" not in err):
get_status, get_err = _request(url, "GET", timeout)
if get_status is not None:
status_code, err = get_status, get_err
elif status_code is None:
status_code, err = get_status, get_err
if status_code is None:
return CheckResult(url=url, status="error", http_status=None, error=err or "unknown")
if 200 <= status_code < 400:
return CheckResult(url=url, status="ok", http_status=status_code, error=None)
return CheckResult(url=url, status="broken", http_status=status_code, error=err or f"http {status_code}")
def run(args: argparse.Namespace) -> int:
if args.path is not None:
path_arg: Path | None = Path(args.path)
if not path_arg.is_absolute():
path_arg = (Path.cwd() / path_arg).resolve()
else:
path_arg = None
skip_env = os.environ.get("LINK_CHECK_SKIP", "")
if skip_env.strip():
skip_domains = {d.strip().lower() for d in skip_env.split(",") if d.strip()}
else:
skip_domains = set(DEFAULT_SKIP_DOMAINS)
files = sorted(set(iter_markdown_files(ROOT, args.phase, path_arg)))
occurrences: dict[str, list[UrlOccurrence]] = {}
for f in files:
try:
text = f.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
continue
try:
rel = f.relative_to(ROOT).as_posix()
except ValueError:
rel = str(f)
for url, line in extract_urls(text):
occurrences.setdefault(url, []).append(UrlOccurrence(file=rel, line=line))
report = Report(checked_files=len(files), unique_urls=len(occurrences))
cache = load_cache()
to_check: list[str] = []
results: dict[str, CheckResult] = {}
for url in occurrences:
if should_skip(url, skip_domains):
report.skipped.append(url)
continue
entry = cache.get(url)
if entry and cache_is_fresh(entry, args.cache):
status = str(entry.get("status", "error"))
http_status = entry.get("http_status")
http_status_int = int(http_status) if isinstance(http_status, (int, float)) else None
err = entry.get("last_error")
results[url] = CheckResult(
url=url,
status=status,
http_status=http_status_int,
error=str(err) if err else None,
cached=True,
)
report.cached_hits += 1
continue
to_check.append(url)
report.requested = len(to_check)
if to_check:
with ThreadPoolExecutor(max_workers=max(1, args.concurrency)) as executor:
futures = {executor.submit(check_url, url, args.timeout): url for url in to_check}
for fut in as_completed(futures):
url = futures[fut]
try:
result = fut.result()
except Exception as exc:
result = CheckResult(
url=url, status="error", http_status=None, error=f"executor: {exc}"
)
results[url] = result
cache[url] = {
"status": result.status,
"http_status": result.http_status,
"checked_at": time.time(),
"last_error": result.error,
}
if not args.json:
mark = "OK" if result.ok else "FAIL"
code = result.http_status if result.http_status is not None else "-"
print(f" [{mark}] {code} {url}", file=sys.stderr)
save_cache(cache)
by_file: dict[str, list[dict[str, object]]] = {}
for url, occs in occurrences.items():
result = results.get(url)
if result is None:
continue
if result.ok:
continue
for occ in occs:
entry = {
"url": url,
"line": occ.line,
"status": result.status,
"http_status": result.http_status,
"error": result.error,
"cached": result.cached,
}
by_file.setdefault(occ.file, []).append(entry)
report.failed.append({"file": occ.file, **entry})
for fname in by_file:
by_file[fname].sort(key=lambda e: (e["line"], e["url"]))
report.by_file = dict(sorted(by_file.items()))
if args.json:
json.dump(report.to_dict(), sys.stdout, indent=2, sort_keys=True)
sys.stdout.write("\n")
else:
print(f"checked {report.checked_files} markdown files", file=sys.stderr)
print(f"unique urls: {report.unique_urls}", file=sys.stderr)
print(f"requested: {report.requested}", file=sys.stderr)
print(f"cache hits: {report.cached_hits}", file=sys.stderr)
print(f"skipped: {len(set(report.skipped))} urls in {len(skip_domains)} domains", file=sys.stderr)
print(f"broken: {len(report.failed)} occurrences across {len(report.by_file)} files", file=sys.stderr)
if report.by_file:
print("", file=sys.stderr)
for fname, entries in report.by_file.items():
print(fname, file=sys.stderr)
for e in entries:
code = e["http_status"] if e["http_status"] is not None else e["error"]
print(f" line {e['line']}: [{code}] {e['url']}", file=sys.stderr)
if args.strict and report.failed:
return 1
return 0
def parse_args(argv: list[str]) -> argparse.Namespace:
parser = argparse.ArgumentParser(
description="Validate external HTTP/HTTPS links across markdown docs.",
)
parser.add_argument("--phase", type=int, default=None, help="restrict to phase NN")
parser.add_argument("--path", default=None, help="restrict to one file or directory")
parser.add_argument("--strict", action="store_true", help="exit 1 if any link is broken")
parser.add_argument("--json", action="store_true", help="emit machine-readable report")
parser.add_argument(
"--timeout",
type=int,
default=DEFAULT_TIMEOUT,
help=f"per-request timeout in seconds (default: {DEFAULT_TIMEOUT})",
)
parser.add_argument(
"--concurrency",
type=int,
default=DEFAULT_CONCURRENCY,
help=f"worker threads (default: {DEFAULT_CONCURRENCY})",
)
parser.add_argument(
"--cache",
type=int,
default=DEFAULT_CACHE_DAYS,
help=f"cache TTL in days; 0 disables (default: {DEFAULT_CACHE_DAYS})",
)
return parser.parse_args(argv)
def main(argv: list[str] | None = None) -> int:
args = parse_args(sys.argv[1:] if argv is None else argv)
return run(args)
if __name__ == "__main__":
sys.exit(main())
+137
View File
@@ -0,0 +1,137 @@
#!/usr/bin/env bash
set -euo pipefail
if [[ $# -lt 2 ]]; then
cat <<'USAGE' >&2
Usage: scripts/scaffold-lesson.sh <phase-dir> <lesson-slug> [title]
Examples:
scripts/scaffold-lesson.sh 05-nlp-foundations-to-advanced 03-tokenizers
scripts/scaffold-lesson.sh 05-nlp-foundations-to-advanced 03-tokenizers "Tokenizers from Scratch"
Creates phases/<phase-dir>/<lesson-slug>/ with code/, notebook/, docs/, outputs/
and a docs/en.md skeleton prefilled from LESSON_TEMPLATE.md.
USAGE
exit 2
fi
PHASE="$1"
LESSON="$2"
TITLE="${3:-}"
REPO_ROOT="$(git rev-parse --show-toplevel 2>/dev/null)"
if [[ -z "$REPO_ROOT" ]]; then
echo "error: run this from inside the ai-engineering-from-scratch git repo" >&2
exit 1
fi
PHASE_DIR="$REPO_ROOT/phases/$PHASE"
LESSON_DIR="$PHASE_DIR/$LESSON"
if [[ ! -d "$PHASE_DIR" ]]; then
echo "error: phase dir not found: phases/$PHASE" >&2
echo " run: ls phases/ to see valid phases" >&2
exit 1
fi
if [[ -e "$LESSON_DIR" ]]; then
echo "error: lesson already exists: phases/$PHASE/$LESSON" >&2
exit 1
fi
if [[ ! "$LESSON" =~ ^[0-9]{2}-[a-z0-9-]+$ ]]; then
echo "error: lesson slug must match NN-kebab-case (e.g. 03-tokenizers)" >&2
exit 1
fi
mkdir -p "$LESSON_DIR/code" "$LESSON_DIR/notebook" "$LESSON_DIR/docs" "$LESSON_DIR/outputs"
PRETTY_TITLE="$TITLE"
if [[ -z "$PRETTY_TITLE" ]]; then
PRETTY_TITLE="$(echo "${LESSON#[0-9][0-9]-}" | tr '-' ' ' | awk '{for (i=1; i<=NF; i++) $i=toupper(substr($i,1,1)) substr($i,2);}1')"
fi
PHASE_NUM="${PHASE%%-*}"
LESSON_NUM="${LESSON%%-*}"
cat >"$LESSON_DIR/docs/en.md" <<EOF
# $PRETTY_TITLE
> [One-line motto. The core idea that sticks.]
**Type:** Build
**Languages:** Python
**Prerequisites:** [prior lessons]
**Time:** ~75 minutes
## The Problem
[2-3 paragraphs. What can't a learner do without this? Make it concrete.]
## The Concept
[Intuition first. Diagrams, tables, mental models. No code yet.]
## Build It
### Step 1: [name]
[explanation]
\`\`\`python
# code here
\`\`\`
### Step 2: [name]
[explanation]
\`\`\`python
# code here
\`\`\`
## Use It
[How a real framework solves the same thing. Compare your version.]
## Ship It
[The reusable artifact this lesson produces. Save in outputs/.]
## Exercises
1. [Easy — reinforce core concept]
2. [Medium — apply to a different problem]
3. [Hard — extend or combine with prior lessons]
## Key Terms
| Term | What people say | What it actually means |
|------|----------------|----------------------|
| | | |
## Further Reading
- []() — []
EOF
cat >"$LESSON_DIR/code/main.py" <<'EOF'
def main():
raise NotImplementedError("implement the lesson")
if __name__ == "__main__":
main()
EOF
touch "$LESSON_DIR/notebook/.gitkeep"
touch "$LESSON_DIR/outputs/.gitkeep"
echo "created phases/$PHASE/$LESSON/"
echo ""
echo "next:"
echo " 1. edit phases/$PHASE/$LESSON/docs/en.md"
echo " 2. write phases/$PHASE/$LESSON/code/main.py"
echo " 3. add a markdown-link row to ROADMAP.md under Phase $PHASE_NUM:"
echo " | $LESSON_NUM | [$PRETTY_TITLE](phases/$PHASE/$LESSON) | ✅ | ~75 min |"
echo " 4. atomic commit: git add phases/$PHASE/$LESSON ROADMAP.md && git commit -m \"feat(phase-$PHASE_NUM/$LESSON_NUM): $PRETTY_TITLE\""
+238
View File
@@ -0,0 +1,238 @@
#!/usr/bin/env python3
"""Scaffold the Agent Workbench pack into a target repository.
Usage:
python3 scripts/scaffold_workbench.py <target_dir> [options]
Options:
--force Overwrite existing AGENTS.md / docs / schemas / scripts.
--minimal Skip docs/ (only AGENTS.md, schemas/, scripts/, VERSION).
--dry-run Print what would happen without writing.
--no-seed Skip seeding starter task_board.json + agent_state.json.
What it installs:
AGENTS.md — root contract for the builder agent
docs/ — agent rules, reviewer rubric, handoff, reliability
schemas/ — JSON Schemas for state + task board + scope
scripts/ — init, run_with_feedback, verify, generate_handoff
task_board.json (seeded) — one todo example task
agent_state.json (seeded) — fresh state record at schema_version 1
.workbench-version — pinned pack version
The pack source is read from this repo at:
phases/14-agent-engineering/42-agent-workbench-capstone/outputs/agent-workbench-pack/
"""
from __future__ import annotations
import argparse
import json
import shutil
import sys
from dataclasses import dataclass
from pathlib import Path
ROOT = Path(__file__).resolve().parent.parent
PACK_DIR = (
ROOT
/ "phases"
/ "14-agent-engineering"
/ "42-agent-workbench-capstone"
/ "outputs"
/ "agent-workbench-pack"
)
REQUIRED_PACK_ENTRIES = ("AGENTS.md", "VERSION", "docs", "schemas", "scripts")
TOP_LEVEL_FILES = ("AGENTS.md",)
TOP_LEVEL_DIRS = ("docs", "schemas", "scripts")
MINIMAL_SKIP_DIRS = ("docs",)
@dataclass
class Action:
kind: str
source: Path | None
target: Path
note: str = ""
def describe(self, target_root: Path) -> str:
rel = self.target.relative_to(target_root)
return f" [{self.kind}] {rel}{('' + self.note) if self.note else ''}"
def validate_pack(pack_dir: Path) -> list[str]:
errors: list[str] = []
if not pack_dir.is_dir():
return [f"pack source not found: {pack_dir}"]
for entry in REQUIRED_PACK_ENTRIES:
if not (pack_dir / entry).exists():
errors.append(f"pack missing required entry: {entry}")
return errors
def plan_copies(target: Path, minimal: bool) -> list[Action]:
actions: list[Action] = []
skip_dirs = set(MINIMAL_SKIP_DIRS) if minimal else set()
for name in TOP_LEVEL_FILES:
actions.append(Action("file", PACK_DIR / name, target / name))
for name in TOP_LEVEL_DIRS:
if name in skip_dirs:
actions.append(Action("skip", None, target / name, "minimal mode"))
continue
actions.append(Action("tree", PACK_DIR / name, target / name))
actions.append(
Action(
"version",
PACK_DIR / "VERSION",
target / ".workbench-version",
)
)
return actions
def detect_collisions(target: Path, actions: list[Action]) -> list[Path]:
collisions: list[Path] = []
for action in actions:
if action.kind == "skip":
continue
if action.target.exists():
collisions.append(action.target)
return collisions
def apply_action(action: Action) -> None:
if action.kind == "skip":
return
if action.kind == "file":
action.target.parent.mkdir(parents=True, exist_ok=True)
shutil.copy2(action.source, action.target)
return
if action.kind == "tree":
shutil.copytree(action.source, action.target, dirs_exist_ok=True)
return
if action.kind == "version":
action.target.parent.mkdir(parents=True, exist_ok=True)
version = action.source.read_text(encoding="utf-8").strip()
action.target.write_text(version + "\n", encoding="utf-8")
return
raise ValueError(f"unknown action kind: {action.kind}")
def seed_task_board(target: Path) -> bool:
path = target / "task_board.json"
if path.exists():
return False
seed = [
{
"id": "T-001",
"goal": "First task. Replace with the real one before the agent starts.",
"owner": "builder",
"acceptance": [
"code change lands",
"tests pass",
"reviewer sign-off recorded",
],
"status": "todo",
}
]
path.write_text(
json.dumps(seed, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
return True
def seed_agent_state(target: Path) -> bool:
path = target / "agent_state.json"
if path.exists():
return False
seed = {
"schema_version": 1,
"active_task_id": None,
"touched_files": [],
"assumptions": [],
"blockers": [],
"next_action": "read AGENTS.md, pick a task from task_board.json, run scripts/init_agent.py",
}
path.write_text(
json.dumps(seed, indent=2, ensure_ascii=False) + "\n",
encoding="utf-8",
)
return True
def render_next_steps(target: Path, pack_version: str) -> str:
rel = target.resolve()
lines = [
"",
f"Workbench pack v{pack_version} scaffolded into {rel}",
"",
"Next steps:",
" 1. Edit task_board.json. Replace T-001 with the real task.",
" 2. Edit AGENTS.md. Set project-specific build cmd, test cmd, deny rules.",
" 3. Run scripts/init_agent.py to capture environment probes.",
" 4. Hand AGENTS.md + task_board.json to the agent. Iterate.",
"",
]
return "\n".join(lines)
def main(argv: list[str]) -> int:
parser = argparse.ArgumentParser(description=__doc__)
parser.add_argument("target_dir", type=Path, help="directory to scaffold into")
parser.add_argument("--force", action="store_true", help="overwrite existing files")
parser.add_argument("--minimal", action="store_true", help="skip docs/")
parser.add_argument("--dry-run", action="store_true", help="preview without writing")
parser.add_argument(
"--no-seed",
action="store_true",
help="do not seed task_board.json / agent_state.json",
)
args = parser.parse_args(argv)
errors = validate_pack(PACK_DIR)
if errors:
for e in errors:
sys.stderr.write(f"error: {e}\n")
return 2
target = args.target_dir
if not target.exists():
if args.dry_run:
sys.stdout.write(f"would create target dir: {target}\n")
else:
target.mkdir(parents=True, exist_ok=True)
actions = plan_copies(target, args.minimal)
collisions = detect_collisions(target, actions)
if collisions and not args.force and not args.dry_run:
sys.stderr.write("error: target already contains:\n")
for c in collisions:
sys.stderr.write(f" {c}\n")
sys.stderr.write("pass --force to overwrite\n")
return 1
pack_version = (PACK_DIR / "VERSION").read_text(encoding="utf-8").strip()
if args.dry_run:
sys.stdout.write(f"dry run — pack v{pack_version}\n")
for action in actions:
sys.stdout.write(action.describe(target) + "\n")
if not args.no_seed:
sys.stdout.write(" [seed] task_board.json (if absent)\n")
sys.stdout.write(" [seed] agent_state.json (if absent)\n")
return 0
for action in actions:
apply_action(action)
if not args.no_seed:
seed_task_board(target)
seed_agent_state(target)
sys.stdout.write(render_next_steps(target, pack_version))
return 0
if __name__ == "__main__":
sys.exit(main(sys.argv[1:]))