wehub/rohitg00--ai-engineering-from-scratch
1
0
Fork 0
Code Issues Pull Requests Actions Deployments Models Agent Notes Packages Projects Releases Wiki Activity
Files
2de7548470d9adaaaa97aa00c6907be8a686383e
rohitg00--ai-engineering-fr…/site/data.js
T
wehub-resource-sync 2de7548470 chore: import upstream snapshot with attribution
2026-07-13 12:09:03 +08:00

12479 lines
639 KiB
JavaScript
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
// Auto-generated by build.js — do not edit manually.
// Last built: 2026-06-25T19:43:10.989Z
const PHASES = [
{
"id": 0,
"name": "Setup & Tooling",
"status": "complete",
"desc": "Get your environment ready for everything that follows.",
"lessons": [
{
"name": "Dev Environment",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/01-dev-environment/",
"summary": "Your tools shape your thinking. Set them up once, set them up right.",
"keywords": "Step 1: System Foundation · Step 2: Python with uv · Step 3: Node.js with pnpm · Step 4: Rust · Step 5: Julia (Optional) · Step 6: GPU Setup (If You Have One) · Step 7: Verify Everything"
},
{
"name": "Git & Collaboration",
"status": "complete",
"type": "Learn",
"lang": "—",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/02-git-and-collaboration/",
"summary": "Version control is not optional. Every experiment, every model, every lesson you build here gets tracked.",
"keywords": "Step 1: Configure git · Step 2: The daily workflow · Step 3: Branching for experiments · Step 4: Working with this course repo"
},
{
"name": "GPU Setup & Cloud",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/03-gpu-setup-and-cloud/",
"summary": "Training on CPU is fine for learning. Training for real needs a GPU.",
"keywords": "Option 1: Local NVIDIA GPU · Option 2: Google Colab · Option 3: Cloud GPU · No GPU? No problem."
},
{
"name": "APIs & Keys",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/04-apis-and-keys/",
"summary": "Every AI API works the same way: send a request, get a response. The details change, the pattern doesn't.",
"keywords": "Step 1: Store API keys safely · Step 2: First API call (Python) · Step 3: First API call (TypeScript) · Step 4: Raw HTTP (no SDK)"
},
{
"name": "Jupyter Notebooks",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/05-jupyter-notebooks/",
"summary": "Notebooks are the lab bench of AI engineering. You prototype here, then move what works into production.",
"keywords": "Step 1: Pick your interface · Step 2: Keyboard shortcuts that matter · Step 3: Cell types · Step 4: Magic commands · Step 5: Display rich output inline · Step 6: Google Colab · Notebooks vs Scripts: When to use which · Common traps"
},
{
"name": "Python Environments",
"status": "complete",
"type": "Build",
"lang": "Shell",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/06-python-environments/",
"summary": "Dependency hell is real. Virtual environments are the cure.",
"keywords": "Option 1: uv venv (Recommended) · Option 2: venv (Built-in) · Option 3: conda (When You Need It) · For This Course: Per-Phase Strategy · 1. Installing globally · 2. Mixing pip and conda · 3. Forgetting to activate · 4. Committing .venv to git · 5. CUDA version mismatch"
},
{
"name": "Docker for AI",
"status": "complete",
"type": "Build",
"lang": "Docker",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/07-docker-for-ai/",
"summary": "Containers make \"works on my machine\" a thing of the past.",
"keywords": "Why AI projects need Docker more than most · Key vocabulary · Common container patterns in AI · Step 1: Install Docker · Step 2: Install NVIDIA Container Toolkit (Linux with NVIDIA GPU) · Step 3: Understand base images · Step 4: Write a Dockerfile for AI development · Step 5: Volume mounts for data and models · Step 6: Docker Compose for multi-service AI apps · Step 7: Useful Docker commands for AI work · No GPU?"
},
{
"name": "Editor Setup",
"status": "complete",
"type": "Build",
"lang": "—",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/08-editor-setup/",
"summary": "Your editor is your co-pilot. Configure it once so it stays out of your way and starts pulling its weight.",
"keywords": "Step 1: Install VS Code · Step 2: Install Essential Extensions · Step 3: Configure Settings · Step 4: Terminal Integration · Step 5: Remote Development (SSH into GPU Boxes) · Cursor · Windsurf · Vim/Neovim"
},
{
"name": "Data Management",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/09-data-management/",
"summary": "Data is the fuel. How you manage it determines how fast you go.",
"keywords": "Step 1: Install the datasets library · Step 2: Load a dataset · Step 3: Stream large datasets · Step 4: Dataset formats · Step 5: Data splits · Step 6: Download and cache models · Step 7: Handle large files · Step 8: Storage patterns"
},
{
"name": "Terminal & Shell",
"status": "complete",
"type": "Learn",
"lang": "—",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/10-terminal-and-shell/",
"summary": "The terminal is where AI engineers live. Get comfortable here.",
"keywords": "Step 1: Know your shell · Step 2: Piping and redirects · Step 3: Background processes · Step 4: tmux · Step 5: Monitoring with htop and nvtop · Step 6: SSH for remote GPU boxes · Step 7: Useful aliases for AI work · Step 8: Common AI terminal patterns"
},
{
"name": "Linux for AI",
"status": "complete",
"type": "Learn",
"lang": "—",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/11-linux-for-ai/",
"summary": "Most AI runs on Linux. You need to know enough to not be stuck.",
"keywords": "Moving Around · Files and Directories · Reading Files · Searching"
},
{
"name": "Debugging & Profiling",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/00-setup-and-tooling/12-debugging-and-profiling/",
"summary": "The worst AI bugs don't crash. They train silently on garbage and report a beautiful loss curve.",
"keywords": "Part 1: Print Debugging (Yes, It Works) · Part 2: Python Debugger (pdb and breakpoint) · Part 3: Python Logging · Part 4: Timing Code Sections · Part 5: cProfile and line_profiler · Part 6: Memory Profiling · Part 7: Common AI Bugs and How to Catch Them · Part 8: TensorBoard Basics · Part 9: VS Code Debugger"
}
]
},
{
"id": 1,
"name": "Math Foundations",
"status": "complete",
"desc": "The intuition behind every AI algorithm, through code.",
"lessons": [
{
"name": "Linear Algebra Intuition",
"status": "complete",
"type": "Learn",
"lang": "Python, Julia",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/01-linear-algebra-intuition/",
"summary": "Every AI model is just matrix math wearing a fancy hat.",
"keywords": "Vectors Are Points (and Directions) · Matrices Are Transformations · The Dot Product Measures Similarity · Linear Independence · Basis and Rank · Projection · Gram-Schmidt Process · Step 1: Vectors from scratch (Python) · Step 2: Matrices from scratch (Python) · Step 3: Why this matters for AI · Step 4: Julia version · Step 5: Linear independence and projection from scratch (Python) · Rank, Projection, and QR with NumPy · PyTorch -- Tensors Are Vectors with Autodiff"
},
{
"name": "Vectors, Matrices & Operations",
"status": "complete",
"type": "Build",
"lang": "Python, Julia",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/02-vectors-matrices-operations/",
"summary": "Every neural network is just matrix multiplication with extra steps.",
"keywords": "Vectors: ordered lists of numbers · Matrices: grids of numbers · Why shapes matter · The operations map · Element-wise vs matrix multiplication · Broadcasting · Step 1: Vector class · Step 2: Matrix class with core operations · Step 3: See it work · Step 4: Connect to neural networks"
},
{
"name": "Matrix Transformations & Eigenvalues",
"status": "complete",
"type": "Build",
"lang": "Python, Julia",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/03-matrix-transformations/",
"summary": "A matrix is a machine that reshapes space. Learn what it does to every point, and you understand the whole transformation.",
"keywords": "Transformations as matrices · Rotation · Scaling · Shearing · Reflection · Composition: chaining transformations · Eigenvalues and eigenvectors · Eigendecomposition · Why eigenvalues matter · Determinant as volume scaling factor · Step 1: Transformation matrices from scratch (Python) · Step 2: Composition of transformations · Step 3: Eigenvalues from scratch (2x2) · Step 4: Determinant as volume scaling factor · 3D rotations with NumPy"
},
{
"name": "Calculus for ML: Derivatives & Gradients",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/04-calculus-for-ml/",
"summary": "Derivatives tell you which way is downhill. That is all a neural network needs to learn.",
"keywords": "What is a derivative? · Partial derivatives: one variable at a time · The gradient: vector of all partial derivatives · The connection to optimization · Numerical vs analytical derivatives · Derivatives by hand for simple functions · The chain rule · The Hessian Matrix · Taylor Series Approximation · Integrals in ML · Multivariable Chain Rule in a Computation Graph · The Jacobian matrix · Why this matters for neural networks · Step 1: Numerical derivative from scratch · Step 2: Partial derivatives and gradients · Step 3: Gradient descent to find the minimum of f(x) = x^2 · Step 4: Gradient descent on a 2D function · Step 5: Comparing numerical and analytical derivatives · Step 6: Computing the Hessian numerically · Step 7: Taylor approximation in action · Step 8: Why this matters for a neural network"
},
{
"name": "Chain Rule & Automatic Differentiation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/05-chain-rule-and-autodiff/",
"summary": "The chain rule is the engine behind every neural network that learns.",
"keywords": "The Chain Rule · Computational Graphs · Forward Mode vs Reverse Mode · Dual Numbers for Forward Mode · Building an Autograd Engine · How PyTorch Autograd Works Under the Hood · Step 1: The Value class · Step 2: Arithmetic operations with gradient tracking · Step 3: The backward pass · Step 4: More operations for a complete engine · Step 5: Mini MLP from scratch · Step 6: Gradient checking · Step 7: Verify against manual calculation · Verify against PyTorch · A more complex expression"
},
{
"name": "Probability & Distributions",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/06-probability-and-distributions/",
"summary": "Probability is the language AI uses to express uncertainty.",
"keywords": "Events, Sample Spaces, and Probability · Conditional Probability and Independence · Probability Mass Functions vs Probability Density Functions · Common Distributions · Expected Value and Variance · Joint and Marginal Distributions · Why the Normal Distribution Shows Up Everywhere · Log Probabilities · Softmax as a Probability Distribution · Sampling · Step 1: Probability basics · Step 2: PMF and PDF from scratch · Step 3: Expected value and variance · Step 4: Sampling from distributions · Step 5: Softmax and log probabilities · Step 6: Central Limit Theorem demonstration · Step 7: Visualization"
},
{
"name": "Bayes' Theorem & Statistical Thinking",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/07-bayes-theorem/",
"summary": "Probability is about what you expect. Bayes' theorem is about what you learn.",
"keywords": "From joint probability to Bayes · The four parts · Medical test example · Spam filter example · Naive Bayes: independence assumption · Maximum likelihood estimation (MLE) · Maximum a posteriori (MAP) · Bayesian vs frequentist: the practical difference · Why Bayesian thinking matters for ML · Step 1: Bayes theorem function · Step 2: Naive Bayes classifier · Step 3: Train on spam data · Step 4: Inspect the learned probabilities · Conjugate Priors · Sequential Bayesian Updating · Connection to A/B Testing"
},
{
"name": "Optimization: Gradient Descent Family",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/08-optimization/",
"summary": "Training a neural network is nothing more than finding the bottom of a valley.",
"keywords": "What optimization means · Gradient descent (vanilla) · Learning rate: the most important hyperparameter · SGD vs batch vs mini-batch · Momentum: the ball rolling downhill · Adam: adaptive learning rates · Learning rate schedules · Convex vs non-convex · Loss landscape visualization · Step 1: Define a test function · Step 2: Vanilla gradient descent · Step 3: SGD with momentum · Step 4: Adam · Step 5: Run and compare"
},
{
"name": "Information Theory: Entropy, KL Divergence",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/09-information-theory/",
"summary": "Information theory measures surprise. Loss functions are built on it.",
"keywords": "Information Content (Surprise) · Entropy (Average Surprise) · Cross-Entropy (The Loss Function You Use Every Day) · KL Divergence (Distance Between Distributions) · Mutual Information · Conditional Entropy · Joint Entropy · Mutual Information (Deep Dive) · Label Smoothing and Cross-Entropy · Why Cross-Entropy Is THE Classification Loss · Bits vs Nats · Perplexity · Step 1: Information content and entropy · Step 2: Cross-entropy and KL divergence · Step 3: Cross-entropy as classification loss · Step 4: Cross-entropy equals negative log-likelihood · Step 5: Mutual information"
},
{
"name": "Dimensionality Reduction: PCA, t-SNE, UMAP",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/10-dimensionality-reduction/",
"summary": "High-dimensional data has structure. You find it by looking from the right angle.",
"keywords": "The curse of dimensionality · PCA: find the directions that matter · Explained variance ratio · Choosing the number of components · t-SNE: preserve neighborhoods · UMAP: faster, better global structure · When to use which · Kernel PCA · Reconstruction Error · Step 1: PCA from scratch · Step 2: Test on synthetic data · Step 3: MNIST digits in 2D · Step 4: Compare with sklearn · Step 5: UMAP comparison"
},
{
"name": "Singular Value Decomposition",
"status": "complete",
"type": "Build",
"lang": "Python, Julia",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/11-singular-value-decomposition/",
"summary": "SVD is the Swiss Army knife of linear algebra. Every matrix has one. Every data scientist needs one.",
"keywords": "What SVD does geometrically · The full decomposition · Left singular vectors, singular values, right singular vectors · Outer product form · Relationship to eigendecomposition · Truncated SVD: low-rank approximation · Image compression with SVD · SVD for recommendation systems · SVD in NLP: Latent Semantic Analysis · SVD for noise reduction · Pseudoinverse via SVD · Numerical stability advantages · Connection to PCA · Step 1: SVD from scratch using power iteration · Step 2: Test and compare with NumPy · Step 3: Image compression demo · Step 4: Noise reduction · Step 5: Pseudoinverse"
},
{
"name": "Tensor Operations",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/12-tensor-operations/",
"summary": "Tensors are the common language between data and deep learning. Every image, every sentence, every gradient flows through them.",
"keywords": "What a tensor is · Tensor shapes in deep learning · How memory layout works · Broadcasting rules · Einsum: the universal tensor operation · Step 1: Tensor storage and strides · Step 2: Reshape, squeeze, unsqueeze · Step 3: Transpose and permute · Step 4: Element-wise operations and reductions · Step 5: Broadcasting with NumPy · Step 6: Einsum operations · Step 7: Attention mechanism via einsum · Scratch vs NumPy · Scratch vs PyTorch · Every neural network layer as a tensor operation"
},
{
"name": "Numerical Stability",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/13-numerical-stability/",
"summary": "Floating point is a leaky abstraction. It will bite you during training, and you will not see it coming.",
"keywords": "IEEE 754: How Computers Store Real Numbers · Why 0.1 + 0.2 != 0.3 · Catastrophic Cancellation · Overflow and Underflow · The Log-Sum-Exp Trick · Why Softmax Needs the Max-Subtraction Trick · NaN and Inf: Detection and Prevention · Numerical Gradient Checking · Mixed Precision Training · bfloat16 vs float16: Why bfloat16 Wins for Training · Gradient Clipping · Normalization Layers as Numerical Stabilizers · Common ML Numerical Bugs · Step 1: Demonstrate floating point precision limits · Step 2: Implement naive vs stable softmax · Step 3: Implement stable log-sum-exp · Step 4: Implement stable cross-entropy · Step 5: Gradient checking · Mixed precision simulation · Gradient clipping · NaN/Inf detection"
},
{
"name": "Norms & Distances",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/14-norms-and-distances/",
"summary": "Your distance function defines what \"similar\" means. Choose wrong and everything downstream breaks.",
"keywords": "Norms: measuring vector magnitude · L1 Norm (Manhattan distance) · L2 Norm (Euclidean distance) · Lp Norms: the general family · L-infinity Norm (Chebyshev distance) · Cosine Similarity and Cosine Distance · Dot Product Similarity vs Cosine Similarity · Mahalanobis Distance · Jaccard Similarity (for sets) · Edit Distance (Levenshtein Distance) · KL Divergence (not a distance, but used like one) · Wasserstein Distance (Earth Mover's Distance) · Why Different Tasks Need Different Distances · Connection to Loss Functions · Connection to Regularization · Nearest Neighbor Search · Step 1: All norm and distance functions · Step 2: Same data, different distances, different neighbors · Step 3: Embedding similarity search"
},
{
"name": "Statistics for ML",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/15-statistics-for-ml/",
"summary": "Statistics is how you know if your model actually works or just got lucky.",
"keywords": "Descriptive Statistics: Summarizing Your Data · Correlation: How Variables Move Together · Covariance Matrix · Hypothesis Testing · The t-test · Chi-squared Test · A/B Testing for ML Models · Statistical Significance vs Practical Significance · Multiple Comparison Problem · Bootstrap Methods · Parametric vs Non-parametric Tests · Central Limit Theorem: Practical Implications · Common Statistical Mistakes in ML Papers"
},
{
"name": "Sampling Methods",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/16-sampling-methods/",
"summary": "Sampling is how AI explores the space of possibilities.",
"keywords": "Why Sampling Matters · Uniform Random Sampling · Inverse CDF Method (Inverse Transform Sampling) · Rejection Sampling · Importance Sampling · Monte Carlo Estimation · Markov Chain Monte Carlo (MCMC): Metropolis-Hastings · Gibbs Sampling · Temperature Sampling (Used in LLMs) · Top-k Sampling · Top-p (Nucleus) Sampling · Reparameterization Trick (Used in VAEs) · Gumbel-Softmax (Differentiable Categorical Sampling) · Stratified Sampling · Connection to Diffusion Models · Step 1: Uniform and inverse CDF sampling · Step 2: Rejection sampling · Step 3: Importance sampling · Step 4: Monte Carlo estimation of pi · Step 5: Metropolis-Hastings MCMC · Step 6: Gibbs sampling · Step 7: Temperature sampling · Step 8: Top-k and top-p sampling · Step 9: Reparameterization trick · Step 10: Gumbel-Softmax"
},
{
"name": "Linear Systems",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/17-linear-systems/",
"summary": "Solving Ax = b is the oldest problem in mathematics that still runs your neural network.",
"keywords": "What Ax = b means geometrically · Column picture vs row picture · Gaussian elimination · Partial pivoting: why it matters · LU decomposition · QR decomposition · Cholesky decomposition · Least squares: when Ax = b has no exact solution · Normal equations = linear regression · Pseudoinverse (Moore-Penrose) · Condition number · Iterative methods: conjugate gradient · The full picture: which method when · Connection to ML · Step 1: Gaussian elimination with partial pivoting · Step 2: LU decomposition · Step 3: Cholesky decomposition · Step 4: Least squares via normal equations · Step 5: Condition number"
},
{
"name": "Convex Optimization",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/18-convex-optimization/",
"summary": "Convex problems have one valley. Neural networks have millions. Knowing the difference matters.",
"keywords": "Convex sets · Convex functions · Testing for convexity · Why convexity matters · Convex vs non-convex in ML · The Hessian matrix · Newton's method · Constrained optimization · Lagrange multipliers · KKT conditions · Regularization as constrained optimization · Duality · Why deep learning works despite non-convexity · Second-order methods in practice · Step 1: Convexity checker · Step 2: Newton's method for 2D · Step 3: Lagrange multiplier solver · Step 4: Compare first-order vs second-order"
},
{
"name": "Complex Numbers for AI",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/19-complex-numbers/",
"summary": "The square root of -1 is not imaginary. It is the key to rotations, frequencies, and half of signal processing.",
"keywords": "What is a complex number? · Complex arithmetic · The complex plane · Polar form · Euler's formula · Why Euler's formula matters for ML · Connection to 2D rotations · Phasors and rotating signals · Roots of unity · Connection to the DFT · Why i is not imaginary · Complex exponentials vs trigonometric functions · Connection to transformers · Step 1: Complex class · Step 2: Polar conversion and Euler's formula · Step 3: Rotation · Step 4: DFT from complex arithmetic · Step 5: Inverse DFT · Step 6: Roots of unity"
},
{
"name": "The Fourier Transform",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/20-fourier-transform/",
"summary": "Every signal is a sum of sine waves. The Fourier transform tells you which ones.",
"keywords": "The DFT definition · What each coefficient means · Inverse DFT · The FFT: making it fast · Spectral analysis · Frequency resolution · The convolution theorem · Windowing · DFT properties · Connection to positional encodings · Connection to CNNs · Spectrograms and the Short-Time Fourier Transform · Aliasing · Zero-padding does not increase resolution · Step 1: DFT from scratch · Step 2: Inverse DFT · Step 3: FFT (Cooley-Tukey) · Step 4: Spectral analysis helpers"
},
{
"name": "Graph Theory for ML",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/21-graph-theory/",
"summary": "Graphs are the data structure of relationships. If your data has connections, you need graph theory.",
"keywords": "Graphs: Nodes and Edges · The Adjacency Matrix · Degree · BFS and DFS · The Graph Laplacian · Spectral Properties · Message Passing · Concepts and ML Applications · Step 1: Graph class from scratch · Step 2: BFS and DFS · Step 3: Connected components and Laplacian eigenvalues · Step 4: Spectral clustering · Step 5: Message passing · numpy spectral analysis"
},
{
"name": "Stochastic Processes",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/01-math-foundations/22-stochastic-processes/",
"summary": "Randomness with structure. The math behind random walks, Markov chains, and diffusion models.",
"keywords": "Random Walks · Markov Chains · Connection to Language Models · Brownian Motion · Langevin Dynamics · MCMC: Markov Chain Monte Carlo · Stochastic Processes in AI · Step 1: Random walk simulator · Step 2: Markov chain · Step 3: Langevin dynamics · Step 4: Metropolis-Hastings · numpy for transition matrices · Connections to real frameworks · Verifying Markov chain convergence"
}
]
},
{
"id": 2,
"name": "ML Fundamentals",
"status": "complete",
"desc": "Classical ML — still the backbone of most production AI.",
"lessons": [
{
"name": "What Is Machine Learning",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/01-what-is-machine-learning/",
"summary": "Machine learning is teaching computers to find patterns in data instead of writing rules by hand.",
"keywords": "Learning From Data, Not Rules · The Three Types of Machine Learning · Beyond the Big Three · Classification vs Regression · The ML Workflow · Training, Validation, and Test Splits · Overfitting vs Underfitting · The Bias-Variance Tradeoff · No Free Lunch Theorem · When NOT to Use Machine Learning · Step 1: Nearest Centroid Classifier from Scratch · Step 2: Train on Synthetic Data · Step 3: Compare Against a Baseline · Why This Matters · Step 4: What the Centroid Classifier Cannot Do"
},
{
"name": "Linear Regression from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/02-linear-regression/",
"summary": "Linear regression draws the best straight line through your data. It is the \"hello world\" of machine learning.",
"keywords": "The Model · The Cost Function (Mean Squared Error) · Gradient Descent · The Normal Equation (Closed-Form Solution) · Multiple Linear Regression · Polynomial Regression · R-Squared Score · Regularization Preview (Ridge Regression) · Step 1: Generate sample data · Step 2: Linear regression from scratch with gradient descent · Step 3: Normal equation (closed-form solution) · Step 4: Multiple linear regression · Step 5: Polynomial regression · Step 6: Ridge regression (L2 regularization)"
},
{
"name": "Logistic Regression & Classification",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/03-logistic-regression/",
"summary": "Logistic regression bends a straight line into an S-curve to answer yes-or-no questions with probabilities.",
"keywords": "Why Linear Regression Fails for Classification · The Sigmoid Function · Logistic Regression = Linear Model + Sigmoid · Binary Cross-Entropy Loss · Gradient Descent for Logistic Regression · The Decision Boundary · Multi-Class Classification with Softmax · Evaluation Metrics · Step 1: Sigmoid function and data generation · Step 2: Logistic regression from scratch · Step 3: Confusion matrix and metrics from scratch · Step 4: Decision boundary analysis · Step 5: Multi-class with softmax · Step 6: Threshold tuning"
},
{
"name": "Decision Trees & Random Forests",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/04-decision-trees/",
"summary": "A decision tree is just a flowchart. But a forest of them is one of the most powerful tools in ML.",
"keywords": "What a decision tree does · Split criteria: measuring impurity · How splitting works · Stopping conditions · Decision trees for regression · Random forests: the power of ensembles · Feature importance · When trees beat neural networks · Step 1: Gini impurity and entropy · Step 2: Find the best split · Step 3: Build the DecisionTree class · Step 4: Build the RandomForest class"
},
{
"name": "Support Vector Machines",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/05-support-vector-machines/",
"summary": "Find the widest street between two classes. That is the entire idea.",
"keywords": "The maximum margin classifier · Support vectors: the critical few · Soft margin: handling noise with the C parameter · Hinge loss: the SVM loss function · Training a linear SVM with gradient descent · The dual formulation and the kernel trick · SVM for regression (SVR) · Why SVMs lost to deep learning (and when they still win) · Step 1: Hinge loss and gradient · Step 2: Linear SVM via gradient descent · Step 3: Kernel functions · Step 4: Margin and support vector identification"
},
{
"name": "KNN & Distance Metrics",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/06-knn-and-distances/",
"summary": "Store everything. Predict by looking at your neighbors. The simplest algorithm that actually works.",
"keywords": "How KNN works · Choosing K · Distance metrics · Weighted KNN · The curse of dimensionality · KD-trees: fast nearest neighbor search · Ball trees: better for moderate dimensions · Lazy learning vs eager learning · KNN for regression · Step 1: Distance functions · Step 2: KNN classifier and regressor · Step 3: KD-tree for efficient search · Step 4: Feature scaling"
},
{
"name": "Unsupervised Learning: K-Means, DBSCAN",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/07-unsupervised-learning/",
"summary": "No labels, no teacher. The algorithm finds structure on its own.",
"keywords": "Clustering: Grouping Similar Things Together · K-Means: The Workhorse · Choosing K · DBSCAN: Density-Based Clustering · Hierarchical Clustering · Gaussian Mixture Models (GMM) · When to Use Which · Anomaly Detection with Clustering · Step 1: K-Means from scratch · Step 2: Elbow method and silhouette score · Step 3: DBSCAN from scratch · Step 4: Gaussian Mixture Model (EM algorithm) · Step 5: Generate test data and run everything"
},
{
"name": "Feature Engineering & Selection",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/08-feature-engineering/",
"summary": "A good feature is worth a thousand data points.",
"keywords": "The Feature Pipeline · Numerical Features · Categorical Features · Text Features · Missing Values · Feature Interaction · Feature Selection · Step 1: Numerical transforms from scratch · Step 2: Categorical encoding from scratch · Step 3: Text features from scratch · Step 4: Missing value imputation from scratch · Step 5: Feature selection from scratch · Step 6: Full pipeline and demo"
},
{
"name": "Model Evaluation: Metrics, Cross-Validation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/09-model-evaluation/",
"summary": "A model is only as good as the way you measure it.",
"keywords": "Train, Validation, Test · K-Fold Cross-Validation · Classification Metrics · Regression Metrics · Learning Curves · Validation Curves · Common Evaluation Mistakes · Step 1: Train/validation/test split · Step 2: K-fold and stratified K-fold cross-validation · Step 3: Confusion matrix and classification metrics · Step 4: Regression metrics · Step 5: Learning curves · Step 6: A simple classifier for testing, plus the full demo"
},
{
"name": "Bias, Variance & the Learning Curve",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/10-bias-variance/",
"summary": "Every model error comes from one of three sources: bias, variance, or noise. You can only control the first two.",
"keywords": "Bias: Systematic Error · Variance: Sensitivity to Training Data · The Decomposition · Model Complexity vs Error · Regularization as Bias-Variance Control · Double Descent: The Modern Perspective · Diagnosing Your Model · Practical Strategies · Ensemble Methods and Variance Reduction · Learning Curves · How to Generate Learning Curves · Step 1: Generate Synthetic Data from a Known Function · Step 2: Bootstrap Sampling and Polynomial Fitting · Step 3: Computing Bias^2, Variance Decomposition · Step 4: Learning Curves · Step 5: Regularization Sweep · Validation Curve: Sweep Model Complexity · Learning Curve: Sweep Training Set Size · Cross-Validation with Regularization Sweep · Putting It All Together: A Complete Diagnostic Workflow"
},
{
"name": "Ensemble Methods: Boosting, Bagging, Stacking",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/11-ensemble-methods/",
"summary": "A group of weak learners, combined correctly, becomes a strong learner. This is not a metaphor. It is a theorem.",
"keywords": "Why Ensembles Work · Bagging (Bootstrap Aggregating) · Boosting (Sequential Error Correction) · AdaBoost · Gradient Boosting · XGBoost: Why It Dominates Tabular Data · Stacking (Meta-Learning) · Voting · Step 1: Decision Stump (Base Learner) · Step 2: AdaBoost from Scratch · Step 3: Gradient Boosting from Scratch · Step 4: Compare against sklearn · When to Use Each Method · The Production Stack for Tabular Data"
},
{
"name": "Hyperparameter Tuning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/12-hyperparameter-tuning/",
"summary": "Hyperparameters are the knobs you turn before training starts. Turning them well is the difference between a mediocre model and a great one.",
"keywords": "Parameters vs Hyperparameters · Grid Search · Random Search · Bayesian Optimization · Early Stopping · Learning Rate Schedulers · Hyperparameter Importance · Practical Strategy · Cross-Validation Integration · Practical Tips · Step 1: Grid Search from Scratch · Step 2: Random Search from Scratch · Step 3: Bayesian Optimization (Simplified) · Step 4: Compare All Methods · Optuna in Practice · Optuna with Pruning · sklearn's Built-in Tuners · Common Mistakes in Hyperparameter Tuning"
},
{
"name": "ML Pipelines & Experiment Tracking",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/13-ml-pipelines/",
"summary": "A model is not a product. A pipeline is. The pipeline is everything from raw data to deployed prediction, and every step must be reproducible.",
"keywords": "What a Pipeline Is · Data Leakage: The Silent Killer · sklearn Pipeline · ColumnTransformer: Different Pipelines for Different Columns · Experiment Tracking · Model Versioning · Data Versioning with DVC · Reproducible Experiments · From Notebook to Production Pipeline · Common Pipeline Mistakes · Step 1: Custom Transformer · Step 2: Pipeline from Scratch · Step 3: Cross-Validation with Pipeline · Step 4: Full Production Pipeline with sklearn"
},
{
"name": "Naive Bayes",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/14-naive-bayes/",
"summary": "The \"naive\" assumption is wrong, and it works anyway. That's the beauty of it.",
"keywords": "Bayes' Theorem (Quick Review) · The Naive Independence Assumption · Why It Still Works · The Math Step by Step · Three Variants · When to Use Each Variant · Laplace Smoothing · Log-Space Computation · Naive Bayes vs Logistic Regression · Classification Pipeline · MultinomialNB · GaussianNB · Demo: Text Classification · Demo: Continuous Features · Prediction Speed · TF-IDF with Naive Bayes · BernoulliNB for Short Text · Calibrating NB Probabilities · Common Gotchas · When Naive Bayes Fails"
},
{
"name": "Time Series Fundamentals",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/15-time-series/",
"summary": "Past performance does predict future results -- if you check for stationarity first.",
"keywords": "What Makes Time Series Different · Components of a Time Series · Stationarity · Autocorrelation · Lag Features: Turning Time Series into Supervised Learning · Walk-Forward Validation · ARIMA Intuition · When to Use What · Forecasting Horizons and Strategies · Common Mistakes in Time Series · Lag Feature Creator · Walk-Forward Cross-Validation · Simple Autoregressive Model · Stationarity Check · Autocorrelation · sklearn TimeSeriesSplit · Evaluation Metrics · Rolling Features · Baselines You Must Beat · Practical Tips"
},
{
"name": "Anomaly Detection",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/16-anomaly-detection/",
"summary": "Normal is easy to define. Abnormal is whatever doesn't fit.",
"keywords": "Types of Anomalies · The Unsupervised Framing · Supervised vs Unsupervised: The Tradeoff · Z-Score Method · IQR Method · Isolation Forest · Local Outlier Factor (LOF) · Comparison · Evaluation Challenges · Anomaly Detection Pipeline · Z-Score Detector · IQR Detector · Isolation Forest from Scratch · Demo Scenarios · sklearn Contamination Parameter · One-Class SVM · Autoencoder Approach (Preview) · Ensemble Anomaly Detection · Production Considerations · Choosing a Threshold · Scaling to Production"
},
{
"name": "Handling Imbalanced Data",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/17-imbalanced-data/",
"summary": "When 99% of your data is \"normal,\" accuracy is a lie.",
"keywords": "Why Accuracy Fails · Better Metrics · The Imbalanced Data Pipeline · SMOTE: Synthetic Minority Oversampling Technique · Sampling Strategies Compared · Class Weights · Threshold Tuning · Cost-Sensitive Learning · Decision Flowchart · Step 1: Generate an imbalanced dataset · Step 2: SMOTE from scratch · Step 3: Random oversampling and undersampling · Step 4: Logistic regression with class weights · Step 5: Threshold tuning · Step 6: Evaluation functions · Step 7: Compare all approaches"
},
{
"name": "Feature Selection",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/02-ml-fundamentals/18-feature-selection/",
"summary": "More features is not better. The right features is better.",
"keywords": "Three Categories of Feature Selection · Variance Threshold · Mutual Information · Recursive Feature Elimination (RFE) · L1 (Lasso) Regularization · Tree-Based Feature Importance · Permutation Importance · Comparison Table · Decision Flowchart · Step 1: Generate synthetic data with known feature structure · Step 2: Variance threshold · Step 3: Mutual information (discrete) · Step 4: Recursive Feature Elimination · Step 5: L1 feature selection · Step 6: Tree-based importance (simple decision tree) · Step 7: Run all methods and compare"
}
]
},
{
"id": 3,
"name": "Deep Learning Core",
"status": "complete",
"desc": "Neural networks from first principles. No frameworks until you build one.",
"lessons": [
{
"name": "The Perceptron: Where It All Started",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/01-the-perceptron/",
"summary": "The perceptron is the atom of neural networks. Split it open and you find weights, a bias, and a decision.",
"keywords": "One Neuron, One Decision · The Decision Boundary · The Learning Rule · The XOR Problem · Step 1: The Perceptron class · Step 2: Train on logic gates · Step 3: Watch XOR fail · Step 4: Solve XOR with two layers · Step 5: Train a Two-Layer Network"
},
{
"name": "Multi-Layer Networks & Forward Pass",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/02-multi-layer-networks/",
"summary": "One neuron draws a line. Stack them, and you can draw anything.",
"keywords": "Layers: Input, Hidden, Output · Neurons and Activations · Forward Pass: How Data Flows · Matrix Dimensions · Universal Approximation Theorem · Composability · Step 1: Sigmoid Activation · Step 2: Layer Class · Step 3: Network Class · Step 4: XOR with Hand-Tuned Weights · Step 5: Circle Classification"
},
{
"name": "Backpropagation from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/03-backpropagation/",
"summary": "Backpropagation is the algorithm that makes learning possible. Without it, neural networks are just expensive random number generators.",
"keywords": "The Chain Rule, Applied to Networks · Computational Graphs · Forward vs Backward · Gradient Flow Through a Network · Vanishing Gradients · Deriving Gradients for a 2-Layer Network · Step 1: The Value Node · Step 2: Operations with Backward Functions · Step 3: Sigmoid and Loss · Step 4: Backward Pass · Step 5: Layer and Network · Step 6: Train on XOR · Step 7: Circle Classification"
},
{
"name": "Activation Functions: ReLU, Sigmoid, GELU & Why",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/04-activation-functions/",
"summary": "Without nonlinearity, your 100-layer network is a fancy matrix multiply. Activations are the gates that let neural networks think in curves.",
"keywords": "Why Nonlinearity Is Necessary · Sigmoid · Tanh · ReLU: The Breakthrough · Leaky ReLU · GELU: The Modern Default · Swish / SiLU · Softmax: The Output Activation · Comparison of Shapes · Gradient Flow Comparison · Which Activation When · Step 1: Implement All Activation Functions with Derivatives · Step 2: Visualize Where Gradients Die · Step 3: Vanishing Gradient Experiment · Step 4: Dead Neuron Detector · Step 5: Training Comparison -- Sigmoid vs ReLU vs GELU"
},
{
"name": "Loss Functions: MSE, Cross-Entropy, Contrastive",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/05-loss-functions/",
"summary": "Your network makes a prediction. The ground truth says otherwise. How wrong is it? That number is the loss. Pick the wrong loss function and your model optimizes for the wrong t…",
"keywords": "Mean Squared Error (MSE) · Cross-Entropy Loss · Why MSE Fails for Classification · Label Smoothing · Contrastive Loss · Focal Loss · Loss Function Decision Tree · Loss Landscape · Step 1: MSE and Its Gradient · Step 2: Binary Cross-Entropy · Step 3: Categorical Cross-Entropy with Softmax · Step 4: Label Smoothing · Step 5: Contrastive Loss (Simplified InfoNCE) · Step 6: MSE vs Cross-Entropy on Classification"
},
{
"name": "Optimizers: SGD, Momentum, Adam, AdamW",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/06-optimizers/",
"summary": "Gradient descent tells you which direction to move. It says nothing about how far or how fast. SGD is a compass. Adam is GPS with traffic data.",
"keywords": "Stochastic Gradient Descent (SGD) · Momentum · RMSProp · Adam: Momentum + RMSProp · AdamW: Weight Decay Done Right · Learning Rate: The Most Important Hyperparameter · Optimizer Comparison · When Each Optimizer Wins · Step 1: Vanilla SGD · Step 2: SGD with Momentum · Step 3: Adam · Step 4: AdamW · Step 5: Training Comparison"
},
{
"name": "Regularization: Dropout, Weight Decay, BatchNorm",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/07-regularization/",
"summary": "Your model gets 99% on training data and 60% on test data. It memorized instead of learning. Regularization is the tax you impose on complexity to force generalization.",
"keywords": "The Overfitting Spectrum · Dropout · Weight Decay (L2 Regularization) · Batch Normalization · Layer Normalization · RMSNorm · Normalization Comparison · Data Augmentation as Regularization · Early Stopping · When to Apply What · Step 1: Dropout (Train and Eval Mode) · Step 2: L2 Weight Decay · Step 3: Batch Normalization · Step 4: Layer Normalization · Step 5: RMSNorm · Step 6: Training With and Without Regularization"
},
{
"name": "Weight Initialization & Training Stability",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/08-weight-initialization/",
"summary": "Initialize wrong and training never starts. Initialize right and 50 layers train as smoothly as 3.",
"keywords": "The Symmetry Problem · Variance Propagation Through Layers · Xavier/Glorot Initialization · Kaiming/He Initialization · Transformer Initialization · Activation Magnitude Through 50 Layers · Choosing the Right Init · Step 1: Initialization Strategies · Step 2: Activation Functions · Step 3: Forward Pass Through 50 Layers · Step 4: The Experiment · Step 5: Symmetry Demonstration · Step 6: Layer-by-Layer Magnitude Report"
},
{
"name": "Learning Rate Schedules & Warmup",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/09-learning-rate-schedules/",
"summary": "The learning rate is the single most important hyperparameter. Not the architecture. Not the dataset size. Not the activation function. The learning rate. If you tune nothing el…",
"keywords": "Constant Learning Rate · Step Decay · Cosine Annealing · Warmup: Why You Start Small · Linear Warmup + Cosine Decay · 1cycle Policy · Schedule Shapes · Decision Flowchart · Real Numbers from Published Models · Step 1: Schedule Functions · Step 2: Visualize All Schedules · Step 3: Training Network · Step 4: Compare All Schedules · Step 5: LR Too High vs Too Low"
},
{
"name": "Build Your Own Mini Framework",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/10-mini-framework/",
"summary": "You have built neurons, layers, networks, backprop, activations, loss functions, optimizers, regularization, initialization, and LR schedules. All as separate pieces. Now wire t…",
"keywords": "The Module Abstraction · Sequential Container · Training vs Evaluation Mode · Optimizer · DataLoader · Framework Architecture · Training Loop · Module Hierarchy · Step 1: Module Base Class · Step 2: Linear Layer · Step 3: Activation Modules · Step 4: Dropout Module · Step 5: BatchNorm Module · Step 6: Sequential Container · Step 7: Loss Functions · Step 8: SGD and Adam Optimizers · Step 9: DataLoader · Step 10: Train a 4-Layer Network on Circle Classification"
},
{
"name": "Introduction to PyTorch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/11-intro-to-pytorch/",
"summary": "You built the engine from pistons and crankshafts. Now learn the one everyone actually drives.",
"keywords": "Why PyTorch Won · Tensors · Autograd · nn.Module · Loss Functions and Optimizers · The Training Loop · Dataset and DataLoader · GPU Training · Comparison: Mini Framework vs PyTorch vs JAX · Step 1: Load MNIST From Raw Files · Step 2: Define the Model · Step 3: Training Loop · Step 4: Wire Everything Together · Quick Comparison: Mini Framework vs PyTorch · Saving and Loading Models · Learning Rate Scheduling"
},
{
"name": "Introduction to JAX",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/12-intro-to-jax/",
"summary": "PyTorch mutates tensors. TensorFlow builds graphs. JAX compiles pure functions. That last one changes how you think about deep learning.",
"keywords": "The JAX Philosophy · jax.numpy: The Familiar Surface · jax.grad: Functional Autodiff · jit: Compile to XLA · vmap: Automatic Vectorization · pmap: Data Parallelism Across Devices · Pytrees: The Universal Data Structure · Functional vs Object-Oriented · The JAX Ecosystem · When to Use JAX vs PyTorch · Random Numbers in JAX · Step 1: Setup and Data · Step 2: Initialize Parameters · Step 3: Forward Pass · Step 4: JIT-Compiled Training Step · Step 5: Training Loop · Flax: The Google Standard · Equinox: The Pythonic Alternative · Optax: Composable Optimizers"
},
{
"name": "Debugging Neural Networks",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/03-deep-learning-core/13-debugging-neural-networks/",
"summary": "Your network compiled. It ran. It produced a number. The number is wrong and nothing crashed. Welcome to the hardest kind of debugging -- the kind where there is no error message.",
"keywords": "The Debugging Mindset · Symptom 1: Loss Not Decreasing · Symptom 2: Loss Decreasing But Model is Bad · Symptom 3: NaN or Inf in Loss · Technique 1: Gradient Checking · Technique 2: Activation Statistics · Technique 3: Gradient Flow Visualization · Technique 4: The Overfit-One-Batch Test · Technique 5: Learning Rate Finder · Common PyTorch Bugs · The Master Debugging Table · Step 1: The NetworkDebugger Class · Step 2: The Overfit-One-Batch Test · Step 3: Learning Rate Finder · Step 4: Gradient Checker · Step 5: Deliberately Broken Networks · PyTorch Built-in Tools · Weights & Biases Integration · TensorBoard · The Debug Checklist (Before Full Training)"
}
]
},
{
"id": 4,
"name": "Computer Vision",
"status": "complete",
"desc": "From pixels to understanding — image, video, 3D, VLMs, and world models.",
"lessons": [
{
"name": "Image Fundamentals: Pixels, Channels, Color Spaces",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/01-image-fundamentals/",
"summary": "An image is a tensor of light samples. Every vision model you will ever use starts from this one fact.",
"keywords": "The full preprocessing pipeline at a glance · A pixel is a sample, not a square · Why three channels · Two layout conventions: HWC and CHW · Byte ranges and dtype · Color spaces and why they exist · Aspect ratio, resizing, and interpolation · Step 1: Load an image and inspect its shape · Step 2: Split channels and re-order layout · Step 3: Grayscale and HSV conversions · Step 4: Normalize, standardize, and reverse it · Step 5: Resize with three interpolation methods"
},
{
"name": "Convolutions from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/02-convolutions-from-scratch/",
"summary": "A convolution is a tiny dense layer you slide across an image, sharing the same weights at every location.",
"keywords": "One kernel, sliding · Output size formula · Padding · Stride · Multiple input channels · The im2col trick · Receptive field · Step 1: Pad an array · Step 2: 2D convolution with nested loops · Step 3: Verify with a hand-designed kernel · Step 4: im2col · Step 5: Fast conv via im2col + matmul · Step 6: A bank of hand-designed kernels"
},
{
"name": "CNNs: LeNet to ResNet",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/03-cnns-lenet-to-resnet/",
"summary": "Every major CNN of the last thirty years is the same conv–nonlinearity–downsample recipe with one new idea bolted on. Learn the ideas in order.",
"keywords": "The four ideas that changed vision · LeNet-5 (1998) · AlexNet (2012) · VGG (2014) · Inception (2014, same year) · The degradation problem · ResNet (2015) · Why residuals matter beyond vision · Step 1: LeNet-5 · Step 2: A VGG block · Step 3: A ResNet BasicBlock · Step 4: A tiny ResNet · Step 5: Compare parameter-to-feature efficiency"
},
{
"name": "Image Classification",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/04-image-classification/",
"summary": "A classifier is a function from pixels to a probability distribution over classes. Everything else is plumbing.",
"keywords": "The classification pipeline · Cross-entropy, logits, and softmax · Why augmentation works · Mixup and cutmix · Label smoothing · Evaluation beyond accuracy · Step 1: A deterministic synthetic dataset · Step 2: Normalisation and augmentation · Step 3: Mixup · Step 4: The training loop · Step 5: Put it together · Step 6: Read the confusion matrix"
},
{
"name": "Transfer Learning & Fine-Tuning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/05-transfer-learning/",
"summary": "Somebody else spent a million GPU hours teaching a network what edges, textures, and object parts look like. You should borrow those features before training your own.",
"keywords": "Feature extraction vs fine-tuning · Why freezing works at all · Discriminative learning rates · The BatchNorm problem · Head design · Layer-wise LR decay · What to evaluate · Step 1: Load a pretrained backbone and inspect it · Step 2: Feature extraction — freeze everything, replace the head · Step 3: Discriminative fine-tuning · Step 4: BatchNorm handling · Step 5: A minimal end-to-end fine-tuning loop · Step 6: Progressive unfreezing"
},
{
"name": "Object Detection — YOLO from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/06-object-detection-yolo/",
"summary": "Detection is classification plus regression, run at every position in a feature map, then cleaned up with non-maximum suppression.",
"keywords": "Detection as dense prediction · Why grids and anchors · Decoding predictions · IoU · Non-maximum suppression · The loss · Detection metrics · Step 1: IoU · Step 2: Non-max suppression · Step 3: Box encoding and decoding · Step 4: A minimal YOLO head · Step 5: Ground-truth assignment · Step 6: The three losses · Step 7: Inference pipeline"
},
{
"name": "Semantic Segmentation — U-Net",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/07-semantic-segmentation-unet/",
"summary": "Segmentation is classification at every pixel. U-Net makes it work by pairing a downsampling encoder with an upsampling decoder and wiring skip connections between them.",
"keywords": "Semantic vs instance vs panoptic · The U-Net shape · Transposed vs bilinear upsample · Cross-entropy on a pixel grid · Dice loss and why you need it · Evaluation metrics · Input resolution trade-off · Step 1: Encoder block · Step 2: Down and up blocks · Step 3: The U-Net · Step 4: Losses · Step 5: IoU metric · Step 6: Synthetic dataset for end-to-end verification · Step 7: Training loop"
},
{
"name": "Instance Segmentation — Mask R-CNN",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/08-instance-segmentation-mask-rcnn/",
"summary": "Add a tiny mask branch to a Faster R-CNN detector and you have instance segmentation. The hard part is RoIAlign, and it is harder than it looks.",
"keywords": "The architecture · Why RoIAlign, not RoIPool · The RPN in one paragraph · The mask head · Losses · Output format · Step 1: RoIAlign from scratch · Step 2: Compare to torchvision's RoIAlign · Step 3: Load a pretrained Mask R-CNN · Step 4: Run inference · Step 5: Swap the heads for a custom class count · Step 6: Freeze what does not need training"
},
{
"name": "Image Generation — GANs",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/09-image-generation-gans/",
"summary": "A GAN is two neural networks in a fixed game. One draws, one critiques. They get better together until the drawings fool the critic.",
"keywords": "The two networks · The game · Non-saturating loss · DCGAN architecture rules · Failure modes and their signatures · Evaluation · Step 1: Generator · Step 2: Discriminator · Step 3: Training step · Step 4: Full training loop on synthetic shapes · Step 5: Sampling · Step 6: Spectral normalisation"
},
{
"name": "Image Generation — Diffusion Models",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/10-image-generation-diffusion/",
"summary": "A diffusion model learns to denoise. Train it to remove a tiny bit of noise from a noisy image, repeat that backwards a thousand times, and you have an image generator.",
"keywords": "The forward process · The closed-form jump · The reverse process · The training loss · The sampler (DDPM) · Why 1000 steps · DDIM: 20x faster sampling · Time conditioning · Step 1: Noise schedule · Step 2: Forward diffusion (q_sample) · Step 3: A tiny time-conditioned U-Net · Step 4: Training loop · Step 5: Sampler (DDPM) · Step 6: DDIM sampler (deterministic, ~20x faster)"
},
{
"name": "Stable Diffusion — Architecture & Fine-Tuning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/11-stable-diffusion/",
"summary": "Stable Diffusion is a DDPM that runs in the latent space of a pretrained VAE, conditioned on text via cross-attention, sampled with a fast deterministic ODE solver, and steered …",
"keywords": "The pipeline · Classifier-free guidance (CFG) · Latent space geometry · The U-Net architecture · LoRA fine-tuning · Schedulers you will see · Step 1: Text-to-image · Step 2: Swap the scheduler · Step 3: Image-to-image · Step 4: Inpainting · Step 5: LoRA loading · Step 6: LoRA training (sketch)"
},
{
"name": "Video Understanding — Temporal Modeling",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/12-video-understanding/",
"summary": "A video is a sequence of images plus the physics that connects them. Every video model either treats time as an extra axis (3D conv), a sequence to attend over (transformer), or…",
"keywords": "The three architectural families · 2D + pool · 3D convolutions · Spatio-temporal transformers · Frame sampling · Evaluation · Datasets you will meet · Step 1: Frame sampler · Step 2: A 2D+pool baseline · Step 3: An I3D-style inflated 3D conv · Step 4: Factorised (2+1)D conv"
},
{
"name": "3D Vision: Point Clouds, NeRFs",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/13-3d-vision-nerf/",
"summary": "3D vision comes in two flavours. Point clouds are the sensor's raw output. NeRFs are the learned volumetric field. Both answer \"what is where in space.\"",
"keywords": "Point clouds · The PointNet architecture · Neural Radiance Fields (NeRFs) · Positional encoding in NeRF · Volumetric rendering · What replaced NeRFs · Datasets and benchmarks · Step 1: PointNet classifier · Step 2: Positional encoding · Step 3: Tiny NeRF MLP · Step 4: Volumetric rendering along a ray"
},
{
"name": "Vision Transformers (ViT)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/14-vision-transformers/",
"summary": "Cut the image into patches, treat each patch as a word, run a standard transformer. Don't look back.",
"keywords": "The pipeline · Patch embedding · Class token · Positional embedding · Transformer encoder block · Why pre-LN · Patch size trade-off · DeiT's recipe for training ViT on ImageNet-1k · Swin vs ConvNeXt · MAE pretraining · Step 1: Patch embedding · Step 2: Transformer block · Step 3: The ViT · Step 4: Sanity check — single image inference"
},
{
"name": "Real-Time Vision: Edge Deployment",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/15-real-time-edge/",
"summary": "Edge inference is the discipline of getting a 90-accuracy model to run at 30 fps on a device with 2 GB of RAM. Every percentage point of accuracy is traded against milliseconds …",
"keywords": "The three budgets · Measurement discipline · FLOPs as a proxy · Quantisation in one paragraph · Pruning and distillation · The inference runtimes · Edge architecture picker · Step 1: Measure latency correctly · Step 2: Parameter and FLOP counts · Step 3: Post-training static quantisation · Step 4: Export to ONNX · Step 5: Benchmark and compare regimes"
},
{
"name": "Build a Complete Vision Pipeline",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/16-vision-pipeline-capstone/",
"summary": "A production vision system is a chain of models and rules stitched with data contracts. The pieces are already in this phase; the capstone wires them together end-to-end.",
"keywords": "The pipeline · Data contracts with Pydantic · Where latency goes · Failure modes · Batching · Step 1: Data contracts · Step 2: A minimal Pipeline class · Step 3: Wire a detector and a classifier · Step 4: FastAPI service · Step 5: Benchmark the pipeline"
},
{
"name": "Self-Supervised Vision — SimCLR, DINO, MAE",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/17-self-supervised-vision/",
"summary": "Labels are the bottleneck of supervised vision. Self-supervised pretraining removes them: learn visual features from 100M unlabelled images, fine-tune on 10k labelled ones.",
"keywords": "Three families · Contrastive learning (SimCLR) · Teacher-student (DINO) · Masked reconstruction (MAE) · Why 75% and not 15% · Linear-probe evaluation · Step 1: Two-view augmentation pipeline · Step 2: InfoNCE loss · Step 3: Sanity check InfoNCE · Step 4: MAE-style masking"
},
{
"name": "Open-Vocabulary Vision — CLIP",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/18-open-vocab-clip/",
"summary": "Train an image encoder and a text encoder together so that matching (image, caption) pairs land at the same point in a shared space. That is the whole trick.",
"keywords": "Two towers · The objective · SigLIP: a better loss · Zero-shot classification · Where CLIP-style models are used in 2026 · Step 1: A tiny two-tower model · Step 2: Contrastive loss · Step 3: Zero-shot classifier · Step 4: Sanity check"
},
{
"name": "OCR & Document Understanding",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/19-ocr-document-understanding/",
"summary": "OCR is a three-stage pipeline — detect text boxes, recognise the characters, then lay them out. Every modern OCR system reorders these stages or merges them.",
"keywords": "The classical pipeline · CTC in one paragraph · Modern end-to-end models · Layout parsing · Evaluation metrics · Step 1: CTC loss + greedy decoder · Step 2: Tiny CRNN recogniser · Step 3: Synthetic OCR · Step 4: Training sketch"
},
{
"name": "Image Retrieval & Metric Learning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/20-image-retrieval-metric/",
"summary": "A retrieval system ranks candidates by a distance in embedding space. Metric learning is the discipline of shaping that space so the distances mean what you want.",
"keywords": "Retrieval at a glance · The four loss families · Triplet loss formally · Cosine similarity vs L2 · Recall@K · FAISS in one paragraph · Instance-level vs category-level retrieval · Step 1: Triplet loss · Step 2: Semi-hard mining · Step 3: Recall@K · Step 4: Putting it together"
},
{
"name": "Keypoint Detection & Pose Estimation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/21-keypoint-pose/",
"summary": "A pose is a set of ordered keypoints. A keypoint detector is a heatmap regressor. Everything else is bookkeeping.",
"keywords": "Top-down vs bottom-up · Heatmap regression · Sub-pixel localisation · Part Affinity Fields (PAFs) · COCO keypoints · 2D vs 3D · Step 1: Gaussian heatmap target · Step 2: Tiny keypoint head · Step 3: Inference — extract keypoint coordinates · Step 4: Synthetic keypoint dataset · Step 5: Training"
},
{
"name": "3D Gaussian Splatting from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/22-3d-gaussian-splatting/",
"summary": "A scene is a cloud of millions of 3D Gaussians. Each one has a position, orientation, scale, opacity, and a colour that depends on viewing direction. Rasterise them, backprop th…",
"keywords": "What a Gaussian carries · Rasterisation, not ray marching · The projection step · The alpha-compositing rule · Why this is differentiable · Densification and pruning · Spherical harmonics in one paragraph · The 2026 production stack · 4D and generative variants · Step 1: A 2D Gaussian · Step 2: 2D splatting rasteriser · Step 3: A trainable 2D splat scene · Step 4: Fit 2D Gaussians to a target image · Step 5: From 2D to 3D · Step 6: Spherical harmonics evaluation"
},
{
"name": "Diffusion Transformers & Rectified Flow",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/23-diffusion-transformers-rectified-flow/",
"summary": "The U-Net is not the secret of diffusion. Replace it with a transformer, swap the noise schedule for a straight-line flow, and suddenly you have SD3, FLUX, and every 2026 text-t…",
"keywords": "From U-Net to transformer · Rectified flow in one paragraph · AdaLN conditioning · Text encoders in SD3 and FLUX · Classifier-free guidance still holds · Consistency, Turbo, Schnell, LCM · Model landscape in 2026 · Why this phase shift matters · Step 1: A DiT block with AdaLN · Step 2: A tiny DiT · Step 3: Rectified flow training · Step 4: Euler sampler · Step 5: End-to-end smoke test"
},
{
"name": "SAM 3 & Open-Vocabulary Segmentation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/24-sam3-open-vocab-segmentation/",
"summary": "Give a model a text prompt and an image and get masks for every matching object. SAM 3 made that a single forward pass.",
"keywords": "The three generations · Promptable Concept Segmentation · Key architectural pieces · Training at scale · SAM 3.1 Object Multiplex · Where Grounded SAM still matters in 2026 · YOLO-World vs SAM 3 · SAM-MI efficiency · Output format for the three models · Step 1: Prompt construction · Step 2: Post-processing helpers · Step 3: A unified open-vocab segmentation interface · Step 4: Hugging Face SAM 3 usage (reference) · Step 5: Measure what Grounded SAM 2 gave you for free"
},
{
"name": "Vision-Language Models (ViT-MLP-LLM)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/25-vision-language-models/",
"summary": "A vision encoder converts an image into tokens. An MLP projector maps those tokens into the LLM's embedding space. A language model does the rest. That pattern — ViT-MLP-LLM — i…",
"keywords": "The ViT-MLP-LLM architecture · DeepStack · Three training stages · Model family comparison (early 2026) · Visual agents · Agentic capabilities + RoPE variants · The alignment problem · Fine-tuning with LoRA / QLoRA · Spatial reasoning is still weak · Step 1: The projector · Step 2: Assemble ViT-MLP-LLM end-to-end · Step 3: CMER computation · Step 4: Toy VLM classifier (runnable)"
},
{
"name": "Monocular Depth & Geometry Estimation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/26-monocular-depth/",
"summary": "A depth map is a single-channel image where each pixel is a distance from the camera. Predicting it from one RGB frame used to be impossible without stereo or LiDAR. In 2026 a f…",
"keywords": "Relative vs metric depth · The encoder-decoder pattern · Why a single image produces depth at all · What monocular depth cannot do · Depth Anything V3 in 2026 · Marigold — diffusion for depth · Intrinsics and the pinhole camera · Evaluation · Step 1: Depth metrics · Step 2: Scale-and-shift alignment · Step 3: Lift depth to a point cloud · Step 4: Smoke test with a synthetic depth scene · Step 5: Depth Anything V3 usage (reference)"
},
{
"name": "Multi-Object Tracking & Video Memory",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/27-multi-object-tracking/",
"summary": "Tracking is detection plus association. Detect every frame. Match this frame's detections to last frame's tracks by ID.",
"keywords": "Tracking-by-detection · Kalman filter in one paragraph · The Hungarian algorithm · ByteTrack's key idea · SAM 2 memory-based tracking · SAM 3.1 Object Multiplex · Three metrics to know · Step 1: IoU-based cost matrix · Step 2: Minimal SORT-style tracker · Step 3: Synthetic trajectory test · Step 4: ID-switch metric"
},
{
"name": "World Models & Video Diffusion",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/04-computer-vision/28-world-models-video-diffusion/",
"summary": "A video model that predicts the next seconds of a scene is a world simulator. Condition that prediction on actions and you have a learned game engine.",
"keywords": "Three families of world-modelling · Video DiT architecture · Conditioning on actions: latent action models · Physical plausibility · Autonomous driving world models · Robotics stack: VLM + video model + inverse dynamics · Evaluation · Model landscape in 2026 · Step 1: 3D patchify for video · Step 2: 3D rotary position encoding · Step 3: Divided attention block · Step 4: Compose a tiny video DiT · Step 5: Check shapes"
}
]
},
{
"id": 5,
"name": "NLP: Foundations to Advanced",
"status": "complete",
"desc": "Language is the interface to intelligence.",
"lessons": [
{
"name": "Text Processing: Tokenization, Stemming, Lemmatization",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/01-text-processing/",
"summary": "Language is continuous. Models are discrete. Preprocessing is the bridge.",
"keywords": "Step 1: a regex word tokenizer · Step 2: a Porter stemmer (step 1a only) · Step 3: a lookup-based lemmatizer · Step 4: pipe them together · NLTK · spaCy · When to pick which · The two failure modes nobody warns you about"
},
{
"name": "Bag of Words, TF-IDF & Text Representation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/02-bag-of-words-tfidf/",
"summary": "Count first, think later. TF-IDF still beats embeddings on well-defined tasks in 2026.",
"keywords": "Step 1: build the vocabulary · Step 2: bag of words · Step 3: term frequency and document frequency · Step 4: TF-IDF · Step 5: L2-normalize rows · When TF-IDF still wins (as of 2026) · When TF-IDF fails · Hybrid: TF-IDF weighted embeddings"
},
{
"name": "Word Embeddings: Word2Vec from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/03-word-embeddings-word2vec/",
"summary": "A word is the company it keeps. Train a shallow net on that idea and geometry falls out.",
"keywords": "Step 1: training pairs from a corpus · Step 2: embedding tables · Step 3: negative sampling objective · Step 4: train on a toy corpus · Step 5: the analogy trick · When Word2Vec still wins in 2026 · Where Word2Vec fails"
},
{
"name": "GloVe, FastText & Subword Embeddings",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/04-glove-fasttext-subword/",
"summary": "Word2Vec trained one embedding per word. GloVe factorized the co-occurrence matrix. FastText embedded the pieces. BPE bridged to transformers.",
"keywords": "GloVe: factorize the co-occurrence matrix · FastText: subword-aware embeddings · BPE: learned subword vocabulary · When to pick which"
},
{
"name": "Sentiment Analysis",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/05-sentiment-analysis/",
"summary": "The canonical NLP task. Most of what you need to know about classical text classification shows up here.",
"keywords": "Step 1: a real mini-dataset · Step 2: multinomial Naive Bayes from scratch · Step 3: logistic regression from scratch · Step 4: handling negation (the failure mode) · Step 5: evaluation metrics that matter · When to reach for a transformer · The reproducibility trap (again)"
},
{
"name": "Named Entity Recognition (NER)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/06-named-entity-recognition/",
"summary": "Pull the names out. Sounds easy until you deal with ambiguous boundaries, nested entities, and domain jargon.",
"keywords": "Step 1: BIO tagging helpers · Step 2: hand-crafted features · Step 3: a simple rule-based + dictionary baseline · Step 4: the CRF step (sketch, not full impl) · Step 5: what a BiLSTM-CRF adds · LLM-based NER (the 2026 option) · Where classical NER still wins · Where it falls apart"
},
{
"name": "POS Tagging & Syntactic Parsing",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/07-pos-tagging-parsing/",
"summary": "Grammar was unfashionable for a while. Then every LLM pipeline needed to validate structured extraction, and it came back.",
"keywords": "Step 1: most-frequent-tag baseline · Step 2: bigram HMM tagger · Step 3: why modern taggers beat this · Step 4: dependency parsing sketch · Where this still matters in 2026"
},
{
"name": "Text Classification — CNNs & RNNs for Text",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/08-cnns-rnns-for-text/",
"summary": "Convolutions learn n-grams. Recurrences remember. Both are superseded by attention. Both still matter on constrained hardware.",
"keywords": "Step 1: TextCNN in PyTorch · Step 2: LSTM classifier · Step 3: the vanishing gradient demo (intuition) · Step 4: why this still was not enough"
},
{
"name": "Sequence-to-Sequence Models",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/09-sequence-to-sequence/",
"summary": "Two RNNs pretending to be a translator. The bottleneck they hit is the reason attention exists.",
"keywords": "Step 1: an encoder · Step 2: a decoder · Step 3: training loop with teacher forcing · Step 4: inference loop (greedy) · Step 5: the bottleneck, demonstrated · When to still reach for RNN-based seq2seq · Exposure bias and its mitigations"
},
{
"name": "Attention Mechanism — The Breakthrough",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/10-attention-mechanism/",
"summary": "The decoder stops squinting at a compressed summary and starts looking at the whole source. Everything after this is attention plus engineering.",
"keywords": "Step 1: additive (Bahdanau) attention · Step 2: Luong dot and general · Step 3: a worked numerical example · Step 4: why this is the bridge to transformers · When classical attention still matters · The attention-weight-as-explanation trap"
},
{
"name": "Machine Translation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/11-machine-translation/",
"summary": "Translation is the task that paid for NLP research for thirty years and keeps paying now.",
"keywords": "Step 1: a pretrained MT call · Step 2: BLEU and chrF · The three-tier evaluation hierarchy (2026) · Step 3: what breaks in production · Step 4: fine-tuning for a domain"
},
{
"name": "Text Summarization",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/12-text-summarization/",
"summary": "Extractive systems tell you what the document said. Abstractive systems tell you what the author meant. Different tasks, different pitfalls.",
"keywords": "Step 1: TextRank (extractive) · Step 2: abstractive with BART · Step 3: ROUGE evaluation · Beyond ROUGE (2026 summarization eval) · Step 4: the factuality problem"
},
{
"name": "Question Answering Systems",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/13-question-answering/",
"summary": "Three systems shaped modern QA. Extractive found spans. Retrieval-augmented grounded them in documents. Generative produced answers. Every modern AI assistant is a mix of the th…",
"keywords": "Step 1: extractive QA with a pretrained model · Step 2: a retrieval-augmented pipeline (sketch) · Step 3: generative with RAG · Step 4: evaluation that reflects the real world · RAGAS: the 2026 production eval framework"
},
{
"name": "Information Retrieval & Search",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/14-information-retrieval-search/",
"summary": "BM25 is precise but brittle. Dense casts a wide net but misses keywords. Hybrid is the 2026 default. Everything else is tuning.",
"keywords": "Step 1: BM25 from scratch · Step 2: dense retrieval with a bi-encoder · Step 3: Reciprocal Rank Fusion · Step 4: hybrid search + rerank · Step 5: evaluation · The hard-won lessons from 2026 production RAG"
},
{
"name": "Topic Modeling: LDA, BERTopic",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/15-topic-modeling/",
"summary": "LDA: documents are mixtures of topics, topics are distributions over words. BERTopic: documents cluster in embedding space, clusters are topics. Same goal, different decompositi…",
"keywords": "Step 1: LDA via scikit-learn · Step 2: BERTopic (production) · Step 3: evaluation"
},
{
"name": "Text Generation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/16-text-generation-pre-transformer/",
"summary": "If a word is surprising, the model is bad. Perplexity makes surprise a number. Smoothing keeps it finite.",
"keywords": "Step 1: trigram counts · Step 2: Laplace smoothing · Step 3: Kneser-Ney (bigram, interpolated) · Step 4: generating text with sampling · Step 5: perplexity"
},
{
"name": "Chatbots: Rule-Based to Neural",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/17-chatbots-rule-to-neural/",
"summary": "ELIZA replied with pattern matches. DialogFlow mapped intents. GPT answered from weights. Claude runs tools and verifies. Each era solved the previous one's worst failure.",
"keywords": "Step 1: rule-based pattern matching · Step 2: retrieval-based (FAQ) · Step 3: neural generation (baseline) · Step 4: LLM agent loop · Step 5: hybrid routing"
},
{
"name": "Multilingual NLP",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/18-multilingual-nlp/",
"summary": "One model, 100+ languages, zero training data for most of them. Cross-lingual transfer is the practical miracle of the 2020s.",
"keywords": "Step 1: zero-shot cross-lingual classification · Step 2: multilingual embedding space · Step 3: few-shot fine-tuning strategy · The tokenization tax (what goes wrong for low-resource languages)"
},
{
"name": "Subword Tokenization: BPE, WordPiece, Unigram, SentencePiece",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/19-subword-tokenization/",
"summary": "Word tokenizers choke on unseen words. Character tokenizers blow up sequence length. Subword tokenizers split the difference. Every modern LLM ships on one.",
"keywords": "Step 1: BPE from scratch · Step 2: encode with the learned merges · Step 3: SentencePiece in practice · Step 4: tiktoken for OpenAI-compatible vocabs"
},
{
"name": "Structured Outputs & Constrained Decoding",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/20-structured-outputs-constrained-decoding/",
"summary": "Ask an LLM for JSON. Get JSON most of the time. In production, \"most\" is the problem. Constrained decoding turns \"most\" into \"always\" by editing the logits before sampling.",
"keywords": "The counterintuitive result · The pitfall that costs you · Step 1: regex-constrained generation from scratch · Step 2: Outlines for JSON Schema · Step 3: Instructor for provider-agnostic Pydantic · Step 4: native vendor APIs"
},
{
"name": "NLI & Textual Entailment",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/21-nli-textual-entailment/",
"summary": "\"t entails h\" means a human reading t would conclude h is true. NLI is the task of predicting entailment / contradiction / neutral. Boring on the surface, load-bearing in produc…",
"keywords": "Step 1: run a pretrained NLI model · Step 2: zero-shot classification · Step 3: faithfulness check for RAG · Step 4: hand-rolled NLI classifier (conceptual)"
},
{
"name": "Embedding Models Deep Dive",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/22-embedding-models-deep-dive/",
"summary": "Word2Vec gave you a vector per word. Modern embedding models give you a vector per passage, cross-lingual, with sparse, dense, and multi-vector views, sized to fit your index. P…",
"keywords": "The MTEB leaderboard tells a partial story · The three-tier pattern · Step 1: baseline — dense embeddings with Sentence-BERT · Step 2: Matryoshka truncation · Step 3: BGE-M3 multi-functionality · Step 4: MTEB eval on a custom task · Step 5: hand-rolled cosine from scratch"
},
{
"name": "Chunking Strategies for RAG",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/23-chunking-strategies-rag/",
"summary": "Chunking configuration influences retrieval quality as much as the choice of embedding model (Vectara NAACL 2025). Get chunking wrong and no amount of reranking saves you.",
"keywords": "The rule that beats every default · Step 1: fixed and recursive chunking · Step 2: semantic chunking · Step 3: parent-document · Step 4: contextual retrieval (Anthropic pattern) · Step 5: evaluate"
},
{
"name": "Coreference Resolution",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/24-coreference-resolution/",
"summary": "\"She called him. He did not answer. The doctor was at lunch.\" Three references to two people and nobody is named. Coreference resolution figures out who is who.",
"keywords": "Step 1: pretrained neural coreference (AllenNLP / spaCy-experimental) · Step 2: rule-based pronoun resolver (teaching) · Step 3: using LLMs for coreference · Step 4: evaluation"
},
{
"name": "Entity Linking & Disambiguation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/25-entity-linking/",
"summary": "NER found \"Paris.\" Entity linking decides: Paris, France? Paris Hilton? Paris, Texas? Paris (the Trojan prince)? Without linking, your knowledge graph stays ambiguous.",
"keywords": "The two measurements · Step 1: build an alias index from Wikipedia redirects · Step 2: context-based disambiguation · Step 3: embedding-based (BLINK-style) · Step 4: generative entity linking (concept) · Step 5: evaluate on AIDA-CoNLL"
},
{
"name": "Relation Extraction & Knowledge Graph Construction",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/26-relation-extraction-kg/",
"summary": "NER found the entities. Entity linking anchored them. Relation extraction finds the edges between them. A knowledge graph is the sum of nodes, edges, and their provenance.",
"keywords": "Step 1: pattern-based extraction · Step 2: supervised relation classification · Step 3: LLM-prompted extraction with anchoring · Step 4: canonicalize onto a closed ontology · Step 5: build a small graph and query"
},
{
"name": "LLM Evaluation: RAGAS, DeepEval, G-Eval",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/27-llm-evaluation-frameworks/",
"summary": "Exact-match and F1 miss semantic equivalence. Human review does not scale. LLM-as-judge is the production answer — with enough calibration to trust the number.",
"keywords": "Step 1: faithfulness with NLI (RAGAS-style) · Step 2: answer relevance · Step 3: G-Eval custom metric · Step 4: CI gate · Step 5: toy eval from scratch"
},
{
"name": "Long-Context Evaluation: NIAH, RULER, LongBench, MRCR",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/28-long-context-evaluation/",
"summary": "Gemini 3 Pro advertises 10M tokens of context. At 1M tokens, 8-needle MRCR drops to 26.3%. Advertised ≠ usable. Long-context evaluation tells you the actual capacity of the mode…",
"keywords": "What to actually report · Step 1: a custom NIAH for your domain · Step 2: a multi-needle variant · Step 3: multi-hop variable tracing (RULER-style) · Step 4: LongBench v2 on your stack"
},
{
"name": "Dialogue State Tracking",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/05-nlp-foundations-to-advanced/29-dialogue-state-tracking/",
"summary": "\"I want a cheap restaurant in the north... actually make it moderate... and add Italian.\" Three turns, three state updates. DST keeps the slot-value dict in sync so the booking …",
"keywords": "The classic failure modes · Step 1: rule-based slot extractor · Step 2: state update loop · Step 3: LLM-driven DST with structured output · Step 4: JGA evaluation · Step 5: handling correction"
}
]
},
{
"id": 6,
"name": "Speech & Audio",
"status": "complete",
"desc": "Hear, understand, speak.",
"lessons": [
{
"name": "Audio Fundamentals: Waveforms, Sampling, FFT",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/01-audio-fundamentals",
"summary": "Waveforms are the raw signal. Spectrograms are the representation. Mel features are the ML-friendly form. Every modern ASR and TTS pipeline walks this ladder, and the first rung…",
"keywords": "Step 1: read a clip and plot the waveform · Step 2: synthesize a sine wave from first principles · Step 3: compute the DFT by hand · Step 4: find the dominant frequency · Step 5: demonstrate aliasing"
},
{
"name": "Spectrograms, Mel Scale & Audio Features",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/02-spectrograms-mel-features",
"summary": "Neural nets do not consume raw waveforms well. They consume spectrograms. They consume mel spectrograms even better. Every ASR, TTS, and audio classifier in 2026 lives or dies b…",
"keywords": "Step 1: frame the waveform · Step 2: Hann window · Step 3: STFT magnitude · Step 4: mel filterbank · Step 5: log-mel · Step 6: MFCCs"
},
{
"name": "Audio Classification",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/03-audio-classification",
"summary": "Everything from \"dog barking vs siren\" to \"which language is this\" is audio classification. The features are mels. The architecture moves each decade. The evaluation stays AUC, …",
"keywords": "Class imbalance is the real challenge · Evaluation · Step 1: featurize · Step 2: fixed-length summary · Step 3: k-NN · Step 4: upgrade to CNN on log-mels · Step 5: the 2026 default — fine-tune BEATs"
},
{
"name": "Speech Recognition (ASR)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/04-speech-recognition-asr",
"summary": "Speech recognition is audio classification at every timestep, glued together by a sequence model that knows English and silence. CTC, RNN-T, and attention are the three ways to …",
"keywords": "WER: the one number · Step 1: greedy CTC decode · Step 2: beam-search CTC · Step 3: WER · Step 4: inference against Whisper · Step 5: streaming with Parakeet or wav2vec 2.0"
},
{
"name": "Whisper: Architecture & Fine-Tuning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/05-whisper-architecture-finetuning",
"summary": "Whisper is a 30-second-window transformer encoder-decoder, trained on 680k hours of multilingual weakly-supervised audio-text pairs. One architecture, multiple tasks, robust acr…",
"keywords": "Variants in 2026 · Fine-tuning · Step 1: run Whisper out of the box · Step 2: chunked long-form · Step 3: fine-tune with LoRA · Step 4: inspect what each layer learns"
},
{
"name": "Speaker Recognition & Verification",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/06-speaker-recognition-verification",
"summary": "ASR asks \"what did they say?\" Speaker recognition asks \"who said it?\" The math looks the same — embeddings plus cosine — but every production decision hinges on a single EER num…",
"keywords": "Scoring · Numbers you should know (2026) · Diarization · Step 1: toy embedding from MFCC statistics · Step 2: cosine similarity + threshold · Step 3: EER from similarity pairs · Step 4: production with SpeechBrain · Step 5: diarize with pyannote"
},
{
"name": "Text-to-Speech (TTS)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/07-text-to-speech",
"summary": "ASR inverts speech to text; TTS inverts text to speech. The 2026 stack is three parts: text → tokens, tokens → mel, mel → waveform. Each part has a default model that fits in a …",
"keywords": "Vocoder evolution · Evaluation · Step 1: phonemize input · Step 2: run Kokoro (2026 CPU default) · Step 3: run F5-TTS with voice cloning · Step 4: HiFi-GAN vocoder from scratch · Step 5: the full pipeline (pseudocode)"
},
{
"name": "Voice Cloning & Voice Conversion",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/08-voice-cloning-conversion",
"summary": "Voice cloning reads your text in someone else's voice. Voice conversion rewrites your voice into someone else's while preserving what you said. Both hang on the same decompositi…",
"keywords": "The ethics bit, not a bolt-on · Numbers (2026) · Step 1: decompose with recognition-synthesis (code-only demo in main.py) · Step 2: zero-shot clone with F5-TTS · Step 3: voice conversion with KNN-VC · Step 4: embed a watermark · Step 5: consent gate"
},
{
"name": "Music Generation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/09-music-generation",
"summary": "2026 music generation: Suno v5 and Udio v4 dominate commercial; MusicGen, Stable Audio Open, and ACE-Step lead open-source. The technical problem is mostly solved. The legal pro…",
"keywords": "Token LM over neural-codec tokens · Diffusion over mels or latents · Hybrid (production) — Suno, Udio, Lyria · Evaluation · Step 1: generate with MusicGen · Step 2: melody conditioning · Step 3: FAD evaluation · Step 4: adding to the LLM-music workflow"
},
{
"name": "Audio-Language Models",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/10-audio-language-models",
"summary": "2026 audio-language models reason over speech + environmental sound + music. Qwen2.5-Omni-7B matches GPT-4o Audio on MMAU-Pro. Audio Flamingo Next beats Gemini 2.5 Pro on LongAu…",
"keywords": "The three-component template · The 2026 model map · Benchmark reality check (2026) · Where LALMs are useful in 2026 · Where they are NOT (yet) useful · Step 1: query Qwen2.5-Omni · Step 2: the projector pattern · Step 3: benchmarking MMAU / LongAudioBench"
},
{
"name": "Real-Time Audio Processing",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/11-real-time-audio-processing",
"summary": "Batch pipelines process a file. Real-time pipelines process the next 20 milliseconds before the next 20 arrive. Every conversational AI, broadcast studio, and telephony bot live…",
"keywords": "Common gotchas · Step 1: ring buffer · Step 2: VAD gate · Step 3: streaming ASR · Step 4: interruption handler"
},
{
"name": "Build a Voice Assistant Pipeline",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/12-voice-assistant-pipeline",
"summary": "Everything from lessons 01-11, stitched together. Build a voice assistant that listens, reasons, and talks back. In 2026 that is a solved engineering problem, not a research pro…",
"keywords": "The seven components · The three failure modes you will hit · 2026 production reference stacks · Step 1: mic capture with chunking (pseudocode) · Step 2: VAD-gated turn capture · Step 3: streaming STT → LLM → TTS · Step 4: tool calling inside the LLM loop · Step 5: interruption handling"
},
{
"name": "Neural Audio Codecs — EnCodec, SNAC, Mimi, DAC",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/13-neural-audio-codecs",
"summary": "2026 audio generation is almost all tokens. EnCodec, SNAC, Mimi, and DAC turn continuous waveforms into discrete sequences that a transformer can predict. The semantic-vs-acoust…",
"keywords": "The core trick: Residual Vector Quantization (RVQ) · The four codecs that matter in 2026 · Frame rates matter for language modeling · Semantic vs acoustic tokens · 2026 reconstruction quality (bits per sec, lower bitrate is better) · Step 1: encode with EnCodec · Step 2: decode and measure reconstruction · Step 3: the semantic-acoustic split (Mimi-style) · Step 4: why AR LM over codec tokens works"
},
{
"name": "Voice Activity Detection & Turn-Taking",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/14-voice-activity-detection-turn-taking",
"summary": "Every voice agent lives or dies on two decisions: is the user speaking now, and are they done? VAD answers the first. Turn-detection (VAD + silence-hangover + semantic endpoint …",
"keywords": "The three-tier VAD cascade · Key parameters and their defaults · The flush trick (Kyutai 2025) · 2026 VAD comparison · Step 1: the energy gate · Step 2: Silero VAD in Python · Step 3: turn-end state machine · Step 4: the flush trick skeleton"
},
{
"name": "Streaming Speech-to-Speech — Moshi, Hibiki",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/15-streaming-speech-to-speech-moshi-hibiki",
"summary": "2024-2026 redefined voice AI. Moshi ships a single model that listens and speaks simultaneously at 200 ms latency. Hibiki does speech-to-speech translation chunk-by-chunk. Both …",
"keywords": "The Moshi architecture · Why inner-monologue text helps · Hibiki: streaming speech-to-speech translation · The broader Kyutai stack (2026) · Sesame CSM — the cousin · 2026 performance numbers · Step 1: the interface · Step 2: the full-duplex loop · Step 3: the training objective (conceptual) · Step 4: where Moshi wins and where it doesn't"
},
{
"name": "Voice Anti-Spoofing & Audio Watermarking",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/16-anti-spoofing-audio-watermarking",
"summary": "Voice cloning shipped faster than defenses. 2026 production voice systems need two things: a detector (AASIST, RawNet2) that classifies real vs fake speech, and a watermark (Aud…",
"keywords": "ASVspoof 5 — the 2024-2025 benchmark · AASIST and RawNet2 — detection model families · AudioSeal — the 2024 watermark default · WavMark · WaveVerify (July 2025) · The gap adversaries exploit · C2PA / Content Authenticity Initiative · Step 1: a simple spectral-feature detector (toy) · Step 2: AudioSeal embed + detect · Step 3: evaluation — EER · Step 4: the production integration"
},
{
"name": "Audio Evaluation — WER, MOS, MMAU, Leaderboards",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/06-speech-and-audio/17-audio-evaluation-metrics",
"summary": "You cannot ship what you cannot measure. This lesson names the 2026 metrics for every audio task: ASR (WER, CER, RTFx), TTS (MOS, UTMOS, SECS, WER-on-ASR-round-trip), audio-lang…",
"keywords": "ASR metrics · TTS metrics · Voice-cloning-specific · Speaker verification · Diarization · Audio classification · Music generation · Audio-language benchmarks · Streaming speech-to-speech · The 2026 leaderboards · Step 1: WER with normalization · Step 2: TTS round-trip WER · Step 3: SECS for voice cloning · Step 4: FAD for music generation · Step 5: EER for speaker verification (same code as Lesson 6)"
}
]
},
{
"id": 7,
"name": "Transformers Deep Dive",
"status": "complete",
"desc": "The architecture that changed everything.",
"lessons": [
{
"name": "Why Transformers: The Problems with RNNs",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/01-why-transformers/",
"summary": "RNNs process tokens one at a time. Transformers process all tokens at once. That single architectural bet changed every scaling curve in deep learning after 2017.",
"keywords": "Step 1: measure serial depth · Step 2: count theoretical operations · Step 3: empirical scaling on long sequences"
},
{
"name": "Self-Attention from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/02-self-attention-from-scratch/",
"summary": "Attention is a lookup table where every word asks \"who matters to me?\" - and learns the answer.",
"keywords": "The Database Lookup Analogy · Q, K, V Computation · The Attention Matrix · Why Scale? · Softmax Turns Scores into Weights · Weighted Sum of Values · Full Pipeline · Step 1: Softmax from scratch · Step 2: Scaled dot-product attention · Step 3: Self-attention class with learned projections · Step 4: Run it on a sentence · Step 5: Visualize attention with ASCII heatmap"
},
{
"name": "Multi-Head Attention",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/03-multi-head-attention/",
"summary": "One attention head learns one relation at a time. Eight heads learn eight. Heads are free. Take more of them.",
"keywords": "Step 1: split heads from the single-head attention we already have · Step 2: run scaled-dot-product attention per head · Step 3: Grouped-Query Attention variant · Step 4: probe what each head learned"
},
{
"name": "Positional Encoding: Sinusoidal, RoPE, ALiBi",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/04-positional-encoding/",
"summary": "Attention is permutation-invariant. \"The cat sat on the mat\" and \"mat the on sat cat the\" produce the same output without positional signal. Three algorithms fix it — each with …",
"keywords": "Absolute sinusoidal · RoPE · ALiBi · What to pick in 2026 · Step 1: sinusoidal encoding · Step 2: RoPE applied to Q, K · Step 3: ALiBi slopes and bias · Step 4: verify relative-distance property of RoPE"
},
{
"name": "The Full Transformer: Encoder + Decoder",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/05-full-transformer/",
"summary": "Attention is the star. Everything else — residuals, normalization, feed-forward, cross-attention — is the scaffolding that lets you stack it deep.",
"keywords": "The six pieces · Encoder block (used by BERT, T5 encoder) · Decoder block (used by GPT, T5 decoder) · Pre-norm vs post-norm · The 2026 modernized block · Parameter count · Step 1: the building blocks · Step 2: wire a 2-layer encoder and a 2-layer decoder · Step 3: run forward on a toy example · Step 4: swap in RMSNorm + SwiGLU"
},
{
"name": "BERT — Masked Language Modeling",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/06-bert-masked-language-modeling/",
"summary": "GPT predicts the next word. BERT predicts a missing word. One sentence of difference — and half a decade of everything embedding-shaped.",
"keywords": "The training signal · The BERT mask rules · Next Sentence Prediction (NSP) — and why it was dropped · What changed in 2026: ModernBERT · Use cases that still pick an encoder in 2026 · Step 1: masking logic · Step 2: run MLM prediction on a tiny corpus · Step 3: compare mask types · Step 4: fine-tune head"
},
{
"name": "GPT — Causal Language Modeling",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/07-gpt-causal-language-modeling/",
"summary": "BERT sees both sides. GPT sees only the past. The triangle mask is the most consequential single line of code in modern AI.",
"keywords": "The mask · Parallel training, serial inference · The loss — shift-by-one · Decoding strategies · What made the \"GPT recipe\" work · Step 1: the causal mask · Step 2: a 2-layer GPT-ish model · Step 3: next-token prediction, end-to-end · Step 4: sampling"
},
{
"name": "T5, BART — Encoder-Decoder Models",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/08-t5-bart-encoder-decoder/",
"summary": "Encoders understand. Decoders generate. Put them back together and you get a model built for input → output tasks: translate, summarize, rewrite, transcribe.",
"keywords": "The forward loop · T5 pretraining — span corruption · BART pretraining — multi-noise denoising · Inference · When to pick each variant in 2026 · Step 1: span corruption · Step 2: verify round-trip · Step 3: BART noising"
},
{
"name": "Vision Transformers (ViT)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/09-vision-transformers/",
"summary": "An image is a grid of patches. A sentence is a grid of tokens. The same transformer eats both.",
"keywords": "Step 1 — patchify · Step 2 — linear embedding · Step 3 — prepend `[CLS]` token, add positional embeddings · Step 4 — standard transformer encoder · Step 5 — head · Variants that mattered · Why it took a while · Step 1: fake image · Step 2: patchify · Step 3: linear embed · Step 4: count parameters for a realistic ViT"
},
{
"name": "Audio Transformers — Whisper Architecture",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/10-audio-transformers-whisper/",
"summary": "Audio is an image of frequency over time. Whisper is a ViT that eats mel spectrograms and speaks back.",
"keywords": "Step 1 — resample + window · Step 2 — convolutional stem · Step 3 — encoder · Step 4 — decoder · Step 5 — task tokens · Step 6 — output · Whisper sizes · What Whisper does not do · 2026 landscape · Step 1: synthesize audio · Step 2: log-mel spectrogram (simplified) · Step 3: pad to 30 s · Step 4: build the prompt tokens"
},
{
"name": "Mixture of Experts (MoE)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/11-mixture-of-experts/",
"summary": "A dense 70B transformer activates every parameter for every token. A 671B MoE activates only 37B per token and beats it on every benchmark. Sparsity is the most important scalin…",
"keywords": "The FFN swap · The load-balancing problem · Shared experts · Fine-grained experts · The cost profile · The catch: memory · Step 1: the router · Step 2: run 100 tokens through the router · Step 3: param count comparison"
},
{
"name": "KV Cache, Flash Attention & Inference Optimization",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/12-kv-cache-flash-attention/",
"summary": "Training is parallel and FLOP-bound. Inference is serial and memory-bound. Different bottleneck, different tricks.",
"keywords": "KV cache math · Flash Attention — the tiling trick · Speculative decoding — the other latency win · Continuous batching · PagedAttention — KV cache as virtual memory · Step 1: KV cache · Step 2: tiled softmax · Step 3: compare naive vs cached decoding on 100-token generation"
},
{
"name": "Scaling Laws",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/13-scaling-laws/",
"summary": "The 2020 Kaplan paper said: bigger model, lower loss. The 2022 Hoffmann paper said: you were under-training. Compute goes into two buckets — parameters and tokens — and the spli…",
"keywords": "The Hoffmann law · Why over-training anyway · Emergence vs smoothness · The 2026 picture · Step 1: Chinchilla loss · Step 2: compute-optimal frontier · Step 3: over-training cost · Step 4: compare to real models"
},
{
"name": "Build a Transformer from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/14-build-a-transformer-capstone/",
"summary": "Thirteen lessons. One model. No shortcuts.",
"keywords": "What we ship · What we don't ship · Target metrics · Step 1: data · Step 2: model · Step 3: training loop · Step 4: sample · Step 5: read the output"
},
{
"name": "Attention Variants — Sliding Window, Sparse, Differential",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/15-attention-variants/",
"summary": "Full attention is a circle. Every token sees every token, and memory pays the price. Four variants bend the shape of the circle and recover half the cost.",
"keywords": "Sliding Window Attention (SWA) · Sparse / Block Attention · Differential Attention (DIFF Transformer, 2024) · Variant Comparison · Step 1: full causal mask (baseline) · Step 2: sliding window causal mask · Step 3: local + strided sparse mask · Step 4: differential attention · Step 5: KV cache sizes"
},
{
"name": "Speculative Decoding — Draft, Verify, Repeat",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/07-transformers-deep-dive/16-speculative-decoding/",
"summary": "Autoregressive decoding is serial. Each token waits for the previous one. Speculative decoding breaks the chain: a cheap model drafts N tokens, the expensive model verifies all …",
"keywords": "The core algorithm · What determines speedup · Medusa — drafts without a draft model · EAGLE — better draft by reusing hidden states · The KV cache dance · Step 1: the rejection step · Step 2: residual distribution · Step 3: one speculative step · Step 4: measure acceptance rate · Step 5: verify distribution equivalence"
}
]
},
{
"id": 8,
"name": "Generative AI",
"status": "complete",
"desc": "Create images, video, audio, 3D, and more.",
"lessons": [
{
"name": "Generative Models: Taxonomy & History",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/01-generative-models-taxonomy-history/",
"summary": "Every image model, text model, video model, and 3D model fits in one of five buckets. Pick the wrong bucket and you will fight the math for weeks. Pick the right one and the fie…"
},
{
"name": "Autoencoders & VAE",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/02-autoencoders-vae/",
"summary": "A plain autoencoder compresses then reconstructs. It memorizes. It does not generate. Add one trick — force the code to look Gaussian — and you get a sampler. That single trick,…",
"keywords": "Step 1: encoder forward · Step 2: reparameterize and decode · Step 3: the ELBO · Step 4: generate"
},
{
"name": "GANs: Generator vs Discriminator",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/03-gans-generator-discriminator/",
"summary": "Goodfellow's trick in 2014 was to skip density entirely. Two networks. One makes fakes. One catches them. They fight until the fakes are indistinguishable from real. It shouldn'…",
"keywords": "Step 1: non-saturating loss · Step 2: one discriminator step per generator step · Step 3: watch for mode collapse"
},
{
"name": "Conditional GANs & Pix2Pix",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/04-conditional-gans-pix2pix/",
"summary": "The first big unlock of 2014-2017 was controlling what a GAN makes. Attach a label, or an image, or a sentence. Pix2Pix did the image version and it still beats every generic te…",
"keywords": "Step 1: append condition to both G and D inputs · Step 2: train conditional · Step 3: verify per-class output"
},
{
"name": "StyleGAN",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/05-stylegan/",
"summary": "Most generators stir `z` into every layer at the same time. StyleGAN split it apart: first map `z` to an intermediate `w`, then *inject* `w` at every resolution level through Ad…",
"keywords": "Step 1: mapping network · Step 2: adaptive instance normalization · Step 3: per-layer noise"
},
{
"name": "Diffusion Models — DDPM from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/06-diffusion-ddpm-from-scratch/",
"summary": "Ho, Jain, Abbeel (2020) gave the field a recipe it could not quit. Destroy the data with noise over a thousand small steps. Train one neural net to predict the noise. Reverse th…",
"keywords": "Step 1: the forward schedule (closed form) · Step 2: sample `x_t` in one shot · Step 3: one training step · Step 4: reverse sampling"
},
{
"name": "Latent Diffusion & Stable Diffusion",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/07-latent-diffusion-stable-diffusion/",
"summary": "Pixel-space diffusion on 512×512 images is a computational war crime. Rombach et al. (2022) noticed that you do not need all 786k dimensions to generate an image — you need enou…",
"keywords": "Step 1: encoder/decoder · Step 2: diffusion in `z`-space · Step 3: classifier-free guidance · Step 4: text conditioning (concept, not code)"
},
{
"name": "ControlNet, LoRA & Conditioning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/08-controlnet-lora-conditioning/",
"summary": "Text alone is a clumsy control signal. ControlNet lets you clone a pretrained diffusion model and steer it with a depth map, pose skeleton, scribble, or edge image. LoRA lets yo…",
"keywords": "ControlNet (Zhang et al., 2023) · LoRA (Hu et al., 2021) · IP-Adapter (Ye et al., 2023) · Step 1: LoRA math · Step 2: zero-init side network"
},
{
"name": "Inpainting, Outpainting & Editing",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/09-inpainting-outpainting-editing/",
"summary": "Text-to-image makes new things. Inpainting fixes old ones. In production, 70% of billable image work is editing — swap a background, remove a logo, extend the canvas, regenerate…",
"keywords": "The naive approach (and why it's wrong) · The proper inpainting model · SDEdit (Meng et al., 2022) — free editing · InstructPix2Pix (Brooks et al., 2023) · RePaint (Lugmayr et al., 2022) · Step 1: 5-D DDPM data · Step 2: train denoiser over all 5 dims · Step 3: at inference, mask-aware reverse · Step 4: outpainting"
},
{
"name": "Video Generation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/10-video-generation/",
"summary": "An image is a 2-D tensor. A video is a 3-D one. The theory is the same; the compute is 10-100x harder. OpenAI's Sora (Feb 2024) proved it was possible. By 2026 Veo 2, Kling 1.5,…",
"keywords": "Patchify · Spatiotemporal DiT · Text conditioning · Training · Step 1: patchify a synthetic 1-D \"video\" · Step 2: position embedding per frame · Step 3: denoiser sees the whole sequence · Step 4: temporal coherence test"
},
{
"name": "Audio Generation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/11-audio-generation/",
"summary": "Audio is a 1-D signal at 16-48 kHz. A five-second clip is 80-240k samples. No transformer attends to that sequence directly. The solution for every production audio model in 202…",
"keywords": "Neural audio codecs · Two generative paradigms on top · Step 1: synthetic audio tokens · Step 2: train a tiny token predictor · Step 3: sample conditionally"
},
{
"name": "3D Generation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/12-3d-generation/",
"summary": "3D is the modality where 2D-to-3D leverage is strongest. The 2023 breakthrough was 3D Gaussian Splatting. The 2024-2026 generative push layers multi-view diffusion + 3D reconstr…",
"keywords": "Representation: 3D Gaussian Splatting (Kerbl et al., 2023) · Multi-view diffusion · Text-to-3D pipelines · NeRF (for context) · Step 1: 2D Gaussian splat · Step 2: render by summing splats · Step 3: fit by gradient descent"
},
{
"name": "Flow Matching & Rectified Flows",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/13-flow-matching-rectified-flows/",
"summary": "Diffusion models take 20-50 sampling steps because they walk a curved path from noise to data. Flow matching (Lipman et al., 2023) and rectified flow (Liu et al., 2022) trained …",
"keywords": "Straight-line flow · Sampling · Rectified flow (Liu 2022) · Why this won for images in 2024 · Step 1: training loss · Step 2: multi-step inference · Step 3: compare step counts"
},
{
"name": "Evaluation: FID, CLIP Score",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/14-evaluation-fid-clip-score/",
"summary": "Every generative model leaderboard cites FID, CLIP score, and a win rate from a human-preference arena. Each number has a failure mode a determined researcher can game. If you d…",
"keywords": "FID — sample quality · CLIP score — prompt adherence · Human preference — the ground truth · Step 1: FID in four lines · Step 2: CLIP-style cosine-similarity · Step 3: Elo aggregation"
},
{
"name": "Visual Autoregressive Modeling (VAR): Next-Scale Prediction",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/08-generative-ai/19-visual-autoregressive-var/",
"summary": "Diffusion models sample iteratively in time (denoising steps). VAR samples iteratively in scale — it predicts a 1x1 token, then 2x2, then 4x4, up to the final resolution, each s…",
"keywords": "VQ-VAE Multi-Scale Tokenizer · Next-Scale Prediction · Generation · Why Next-Scale Wins Over Next-Token · Scaling Law · Relationship to Diffusion"
}
]
},
{
"id": 9,
"name": "Reinforcement Learning",
"status": "complete",
"desc": "The foundation of RLHF and game-playing AI.",
"lessons": [
{
"name": "MDPs, States, Actions & Rewards",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/01-mdps-states-actions-rewards/",
"summary": "A Markov Decision Process is five things: states, actions, transitions, rewards, a discount. Everything in RL — Q-learning, PPO, DPO, GRPO — optimizes over this shape. Learn it …",
"keywords": "Step 1: a tiny deterministic MDP · Step 2: roll out a policy · Step 3: compute `V^π` exactly via the Bellman equation · Step 4: `γ` is a hyperparameter with physical meaning"
},
{
"name": "Dynamic Programming",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/02-dynamic-programming/",
"summary": "Dynamic programming is RL with cheating. You already know the transition and reward functions; you just iterate the Bellman equation until `V` or `π` stops moving. It is the ben…",
"keywords": "Step 1: build the GridWorld MDP model · Step 2: policy evaluation · Step 3: policy improvement · Step 4: stitch them together · Step 5: value iteration (the one-loop version)"
},
{
"name": "Monte Carlo Methods",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/03-monte-carlo-methods/",
"summary": "Dynamic programming needs a model. Monte Carlo needs nothing but episodes. Run the policy, watch the returns, average them. The simplest idea in RL — and the one that unlocks ev…",
"keywords": "Step 1: rollout → list of (s, a, r) · Step 2: compute returns (reverse sweep) · Step 3: first-visit MC evaluation · Step 4: ε-greedy MC control (on-policy) · Step 5: compare to DP gold standard"
},
{
"name": "Q-Learning, SARSA",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/04-q-learning-sarsa/",
"summary": "Monte Carlo waits until the episode ends. TD updates after every step by bootstrapping the next value estimate. Q-learning is off-policy and optimistic; SARSA is on-policy and c…",
"keywords": "Step 1: SARSA on ε-greedy policy · Step 2: Q-learning · Step 3: learning curves · Step 4: compare to DP truth"
},
{
"name": "Deep Q-Networks (DQN)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/05-dqn/",
"summary": "2013: Mnih trained one Q-learning network on raw pixels, beat every classical RL agent on seven Atari games. 2015: extended to 49 games, published in Nature, sparked the deep-RL…",
"keywords": "Step 1: replay buffer · Step 2: a tiny Q-network (manual MLP) · Step 3: the DQN update · Step 4: the outer loop"
},
{
"name": "Policy Gradients — REINFORCE",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/06-policy-gradients-reinforce/",
"summary": "Stop estimating value. Parameterize the policy directly, compute the gradient of expected return, step uphill. Williams (1992) wrote it in one theorem. It is why PPO, GRPO, and …",
"keywords": "Step 1: softmax policy network · Step 2: sampling and log-probability · Step 3: rollout with log-probs captured · Step 4: REINFORCE update · Step 5: baselines"
},
{
"name": "Actor-Critic — A2C, A3C",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/07-actor-critic-a2c-a3c/",
"summary": "REINFORCE is noisy. Add a critic that learns `V̂(s)`, subtract it from the return, and you get an advantage that has the same expectation but far lower variance. That is actor-c…",
"keywords": "Step 1: a critic · Step 2: n-step advantage · Step 3: combined update · Step 4: parallelization (A3C vs A2C)"
},
{
"name": "PPO",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/08-ppo/",
"summary": "A2C throws away each rollout after one update. PPO wraps the policy gradient in a clipped importance ratio so you can do 10+ epochs on the same data without the policy exploding…",
"keywords": "Step 1: capture `log π_old(a | s)` at rollout time · Step 2: compute GAE advantages (Lesson 07) · Step 3: clipped surrogate update · Step 4: value and entropy · Step 5: diagnostics"
},
{
"name": "Reward Modeling & RLHF",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/09-reward-modeling-rlhf/",
"summary": "Humans cannot write a reward function for \"good assistant response,\" but they can compare two responses and pick the better one. Fit a reward model to those comparisons, then RL…",
"keywords": "Step 1: synthetic preference data · Step 2: Bradley-Terry reward model · Step 3: PPO-like policy on top of RM · Step 4: monitor the KL · Step 5: the production recipe with TRL"
},
{
"name": "Multi-Agent RL",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/10-multi-agent-rl/",
"summary": "Single-agent RL assumes the environment is stationary. Put two learning agents in the same world and that assumption breaks: each agent is part of the other's environment, and b…",
"keywords": "Step 1: the multi-agent env · Step 2: independent Q-learning · Step 3: centralized Q with decomposed-value update · Step 4: simple self-play (adversarial 2-agent)"
},
{
"name": "Sim-to-Real Transfer",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/11-sim-to-real-transfer/",
"summary": "A policy trained in a simulator that fails on hardware is a policy that memorized the simulator. Domain randomization, domain adaptation, and system identification are the three…",
"keywords": "Step 1: parameterized sim · Step 2: train with DR · Step 3: evaluate zero-shot on \"real\" slips · Step 4: compare to narrow training"
},
{
"name": "RL for Games",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/09-reinforcement-learning/12-rl-for-games/",
"summary": "1992: TD-Gammon beat human champions at backgammon with pure TD. 2016: AlphaGo beat Lee Sedol. 2017: AlphaZero dominated chess, shogi, and Go from scratch. 2024: DeepSeek-R1 pro…",
"keywords": "Step 1: a tiny verifier environment · Step 2: policy: softmax over K answer tokens per prompt · Step 3: group sampling and group-relative advantage · Step 4: compare to REINFORCE baseline (value-free) · Step 5: observe entropy and KL"
}
]
},
{
"id": 10,
"name": "LLMs from Scratch",
"status": "complete",
"desc": "Build, train, and understand large language models.",
"lessons": [
{
"name": "Tokenizers: BPE, WordPiece, SentencePiece",
"status": "complete",
"type": "Build",
"lang": "Python, Rust",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/01-tokenizers/",
"summary": "Your LLM does not read English. It reads integers. The tokenizer decides whether those integers carry meaning or waste it.",
"keywords": "Three Approaches That Failed (and One That Won) · BPE: Byte Pair Encoding · Byte-Level BPE (GPT-2, GPT-3, GPT-4) · WordPiece (BERT) · SentencePiece (Llama, T5) · Vocabulary Size Tradeoffs · The Multilingual Tax · Step 1: Character-Level Tokenizer · Step 2: BPE Tokenizer from Scratch · Step 3: Encode and Decode Roundtrip · Step 4: Compare with tiktoken · Step 5: Vocabulary Analysis · tiktoken (OpenAI) · Hugging Face tokenizers · Loading Llama's Tokenizer"
},
{
"name": "Building a Tokenizer from Scratch",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/02-building-a-tokenizer/",
"summary": "Lesson 01 gave you a toy. This lesson gives you a weapon.",
"keywords": "The Full Pipeline · Byte-Level BPE · Pre-Tokenization · Special Tokens · Chat Templates · Speed · Step 1: Byte-Level Encoding · Step 2: Pre-Tokenizer with Regex · Step 3: BPE on Byte Sequences · Step 4: Special Token Handling · Step 5: Full Tokenizer Class · Step 6: Multilingual Test · Comparing Real Tokenizers"
},
{
"name": "Data Pipelines for Pre-Training",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/03-data-pipelines/",
"summary": "The model is a mirror. It reflects whatever data you feed it. Feed it garbage, it reflects garbage with perfect fluency.",
"keywords": "Where the Data Comes From · Data Cleaning · Deduplication with MinHash · Sequence Packing · The Chinchilla Scaling Law · Step 1: Text Cleaning · Step 2: MinHash Deduplication · Step 3: Tokenize and Pack Sequences · Step 4: DataLoader for Training · Step 5: Dataset Statistics · Compare With HuggingFace Datasets"
},
{
"name": "Pre-Training a Mini GPT (124M)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/04-pre-training-mini-gpt/",
"summary": "GPT-2 Small has 124 million parameters. That's 12 transformer layers, 12 attention heads, and 768-dimensional embeddings. You can train it from scratch on a single GPU in a few …",
"keywords": "The GPT Architecture · The Transformer Block · Attention: The Core Mechanism · KV Cache: Why Inference Is Fast · Prefill vs Decode: Two Phases of Inference · The Training Loop · GPT-2 Small: The Numbers · Step 1: Embedding Layer · Step 2: Self-Attention with Causal Mask · Step 3: Multi-Head Attention · Step 4: Transformer Block · Step 5: Full GPT Model · Step 6: Training Loop · Step 7: Text Generation · Full Training and Generation Demo"
},
{
"name": "Distributed Training, FSDP, DeepSpeed",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/05-scaling-distributed/",
"summary": "Your 124M model trained on one GPU. Now try 7 billion parameters. The model doesn't fit in memory. The data takes weeks on a single machine. Distributed training isn't optional …",
"keywords": "Why Distribution is Required · Data Parallelism · Tensor Parallelism · Pipeline Parallelism · FSDP: Fully Sharded Data Parallel · DeepSpeed ZeRO · Mixed Precision Training · Megatron-LM and 3D Parallelism · Step 1: Simulate Data Parallelism · Step 2: Simulate Tensor Parallelism · Step 3: Simulate Pipeline Parallelism · Step 4: Memory Calculator · Step 5: Mixed Precision Simulation · Run All Simulations"
},
{
"name": "Instruction Tuning — SFT",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/06-instruction-tuning-sft/",
"summary": "A base model predicts the next token. That's it. It doesn't follow instructions, answer questions, or refuse harmful requests. SFT is the bridge between a token predictor and a …",
"keywords": "What SFT Actually Does · Data Formats · Why It Works · The Masked Loss · Training Hyperparameters · Catastrophic Forgetting · Real Numbers · Step 1: Instruction Dataset · Step 2: Tokenize with Chat Template · Step 3: Masked Cross-Entropy Loss · Step 4: SFT Training Loop · Step 5: Compare Base vs SFT Model · Step 6: Measure Catastrophic Forgetting · Full SFT Pipeline Demo"
},
{
"name": "RLHF — Reward Model + PPO",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/07-rlhf/",
"summary": "SFT teaches the model to follow instructions. But it doesn't teach the model which response is BETTER. Two grammatically correct, factually accurate answers can differ enormousl…",
"keywords": "The Three Stages · The Reward Model · PPO: Proximal Policy Optimization · The PPO Objective in Detail · Reward Hacking · Real RLHF Pipelines · Step 1: Synthetic Preference Data · Step 2: Reward Model Architecture · Step 3: Bradley-Terry Loss · Step 4: Simplified PPO Loop · Step 5: Reward Score Comparison · Full RLHF Pipeline Demo"
},
{
"name": "DPO — Direct Preference Optimization",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/08-dpo/",
"summary": "RLHF works. It also requires training three models (SFT, reward model, policy), managing PPO's instability, and tuning a KL penalty. DPO asks: what if you could skip all of that…",
"keywords": "The Key Insight · The DPO Loss · Why DPO is Simpler · When DPO Beats RLHF · When RLHF Beats DPO · Beyond DPO: KTO, ORPO, SimPO · Real DPO Deployments · Step 1: Preference Dataset · Step 2: Sequence Log-Probability · Step 3: The DPO Loss · Step 4: DPO Training Loop · Step 5: Compare DPO vs RLHF · Step 6: Beta Sensitivity Analysis · Full DPO Pipeline Demo"
},
{
"name": "Constitutional AI & Self-Improvement",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/09-constitutional-ai-self-improvement/",
"summary": "RLHF needs humans in the loop. Constitutional AI replaces most of them with the model itself. Write a list of principles, have the model critique its own outputs against those p…",
"keywords": "The Constitutional AI Loop · What the Constitution Actually Does · GRPO: Group-Relative Policy Optimization · Why GRPO Matters for Reasoning · Process Reward Models vs Outcome Reward Models · Self-Improvement: The Feedback Multiplier · When To Use What · Step 1: The Constitution · Step 2: Self-Critique and Revise · Step 3: Rule-Based Rewards · Step 4: Group-Relative Advantage · Step 5: GRPO Update · Step 6: Self-Improvement Round"
},
{
"name": "Evaluation — Benchmarks, Evals",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/10-evaluation/",
"summary": "Goodhart's Law: when a measure becomes a target, it ceases to be a good measure. Every frontier lab games benchmarks. MMLU scores go up while models still can't reliably count t…",
"keywords": "The Eval Landscape · Why Benchmarks Break · Perplexity: A Quick Health Check · LLM-as-Judge · ELO Ratings from Pairwise Comparisons · Eval Frameworks · Building Custom Evals · Step 1: A Minimal Eval Framework · Step 2: Scoring Functions · Step 3: ELO Rating System · Step 4: Perplexity Calculation · Step 5: Aggregate Results · Step 6: Run the Full Pipeline · Step 7: ELO Tournament · Step 8: Perplexity Comparison · lm-evaluation-harness (EleutherAI) · promptfoo · RAGAS for RAG evaluation"
},
{
"name": "Quantization: INT8, GPTQ, AWQ, GGUF",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/11-quantization/",
"summary": "A 70B model in FP16 needs 140GB. Two A100s just for weights. Quantize to FP8: one 80GB GPU. INT4: a MacBook.",
"keywords": "Number Formats: What Each Bit Does · How Quantization Works · Sensitivity Hierarchy · PTQ vs QAT · GPTQ, AWQ, GGUF · Quality Measurement · Real Numbers · Step 1: Number Format Representations · Step 2: Symmetric Quantization (Per-Tensor and Per-Channel) · Step 3: Quality Measurement · Step 4: Bit-Width Sweep · Step 5: Sensitivity Experiment · Step 6: Simulated GPTQ · Step 7: AWQ Simulation · Step 8: Full Pipeline · Quantizing with AutoGPTQ · Quantizing with AutoAWQ · Converting to GGUF · Serving with vLLM"
},
{
"name": "Inference Optimization",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/12-inference-optimization/",
"summary": "Two phases define LLM inference. Prefill processes your prompt in parallel -- compute-bound. Decode generates tokens one at a time -- memory-bound. Every optimization targets on…",
"keywords": "Prefill vs Decode · KV Cache · Continuous Batching · PagedAttention · Speculative Decoding · Prefix Caching · Inference Engines · The Ops:Byte Framework · Step 1: KV Cache from Scratch · Step 2: Attention with KV Cache · Step 3: Continuous Batching Simulator · Step 4: Prefix Cache · Step 5: Speculative Decoding Simulator · Step 6: KV Cache Memory Profiler"
},
{
"name": "Building a Complete LLM Pipeline",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/13-building-complete-llm-pipeline/",
"summary": "Everything from Lessons 01 to 12 is one stage of one pipeline. This lesson is the scaffold that turns those stages into a single end-to-end run: tokenize, pre-train, scale, SFT,…",
"keywords": "The Twelve Stages · The Manifest · Artifact Typing · The Eval Gate · The Orchestrator · Experiment Tracking and Artifact Storage · Costing · Reproducibility vs Determinism · Rollback Plan · Production Recipes Observed in 2026"
},
{
"name": "Open Models: Architecture Walkthroughs",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/14-open-models-architecture-walkthroughs/",
"summary": "You built a GPT-2 Small from scratch in Lesson 04. Frontier open models in 2026 are the same family with five or six concrete changes. RMSNorm instead of LayerNorm. SwiGLU inste…",
"keywords": "The Invariant Core · The Six Knobs That Actually Move · Knob 1: RMSNorm · Knob 2: RoPE · Knob 3: SwiGLU · Knob 4: Attention Head Sharing · Knob 5: Mixture of Experts · Knob 6: Pre-norm stays · Model-by-Model Diff · Reading a config.json · Activation memory budget · KV Cache budget · When Each Model Wins"
},
{
"name": "Speculative Decoding and EAGLE-3",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/15-speculative-decoding-eagle3/",
"summary": "Phase 7 · Lesson 16 proved the math: the Leviathan rejection rule preserves the verifier's distribution exactly. This lesson is the training-stack view of 2026 production specul…",
"keywords": "The invariant: Leviathan rejection sampling · What determines speedup · The two-year progression · KV cache rollback · Draft architectures in 2026 · Step 1: the rejection rule · Step 2: residual distribution · Step 3: a full speculative step · Step 4: KV rollback bookkeeping · Step 5: the Leviathan check · Step 6: speedup vs. α"
},
{
"name": "Differential Attention (V2)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/16-differential-attention-v2/",
"summary": "Softmax attention spreads a small amount of probability over every non-matching token. Over 100k tokens that noise adds up and drowns the signal. Differential Transformer (Ye et…",
"keywords": "The noise floor of softmax · The differential idea · Why this matches headed noise-canceling · V1 vs V2: the diff · When to reach for it · How it stacks with other 2026 knobs · Step 1: standard softmax attention · Step 2: split Q, K into two halves · Step 3: two softmax branches + subtraction · Step 4: noise cancellation measurement · Step 5: V1 vs V2 parameter accounting"
},
{
"name": "Native Sparse Attention (DeepSeek NSA)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/17-native-sparse-attention/",
"summary": "At 64k tokens, attention eats 70-80% of decode latency. Every open-model lab has a plan to fix it. DeepSeek's NSA (ACL 2025 best paper) is the one that stuck: three parallel att…",
"keywords": "Three parallel branches · Why this is \"natively trainable\" · Hardware-aligned kernel · The compute budget · How does it compare · Step 1: compress tokens into blocks · Step 2: compressed-branch attention · Step 3: top-k block selection · Step 4: sliding-window attention · Step 5: gate + combine · Step 6: compute counting"
},
{
"name": "Multi-Token Prediction (MTP)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/18-multi-token-prediction/",
"summary": "Every autoregressive LLM from GPT-2 to Llama 3 trains on one loss per position: predict the next token. DeepSeek-V3 added a second loss per position: predict the token after tha…",
"keywords": "The sequential MTP recipe · Why sequential, not parallel · Parameter accounting · The speculative-decoding payoff · Relation to EAGLE · Step 1: shared embedding table · Step 2: the per-depth combination · Step 3: the transformer block at depth k · Step 4: the shared output head · Step 5: per-depth loss · Step 6: parameter accounting"
},
{
"name": "DualPipe Parallelism",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/19-dualpipe-parallelism/",
"summary": "DeepSeek-V3 was trained on 2,048 H800 GPUs with MoE experts scattered across nodes. Cross-node expert all-to-all communication cost 1 GPU-hour of comm for every 1 GPU-hour of co…",
"keywords": "Pipeline parallelism refresher · Idea 1: chunk decomposition · Idea 2: bidirectional scheduling · A hand-traced schedule · Bubble accounting · DualPipeV — the refinement · What it means for a 14.8T-token run · Where it sits in the stack"
},
{
"name": "DeepSeek-V3 Architecture Walkthrough",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/20-deepseek-v3-walkthrough/",
"summary": "Phase 10 · Lesson 14 named the six architectural knobs every open model turns. DeepSeek-V3 (December 2024, 671B parameters total, 37B active) turns all six and adds four more: M…",
"keywords": "The invariant core, again · The twist: MLA instead of GQA · The routing: auxiliary-loss-free load balancing · The MTP: denser training + free draft · The training: DualPipe · The config, field by field · Parameter accounting · The 671B / 37B ratio · Where DeepSeek-V3 sits · The follow-on: R1, V4"
},
{
"name": "Jamba — Hybrid SSM-Transformer",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/21-jamba-hybrid-ssm-transformer/",
"summary": "State space models (SSMs) and transformers want different things. Transformers buy quality via attention at quadratic cost. SSMs buy linear-time inference and constant memory vi…",
"keywords": "An SSM in one page · The Jamba block · Why the 1:7 ratio · Positional encoding · The memory budget · Mamba-3: the pure-SSM baseline in 2026 · When to reach for a hybrid · The competitive landscape"
},
{
"name": "Async and Hogwild! Inference",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/22-async-hogwild-inference/",
"summary": "Speculative decoding (Phase 10 · 15) parallelizes tokens within one sequence. Multi-agent frameworks parallelize across whole sequences but force explicit coordination (voting, …",
"keywords": "The setup · Why coordination emerges · The naming · RoPE makes this tractable · Wall-time math · Concrete example · When to reach for Hogwild! · When not to · The experimental status · Step 1: the shared cache · Step 2: the worker loop · Step 3: the coordination heuristic · Step 4: measured speedup · Step 5: stress the coordination"
},
{
"name": "Speculative Decoding and EAGLE",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/25-speculative-decoding/",
"summary": "A frontier LLM generating one token requires a full forward pass over billions of parameters. That forward pass is massively over-provisioned: most of the time a much smaller mo…",
"keywords": "The Two-Model Setup · The Exactness Rule · Expected Speedup · Training the Draft: Distillation · EAGLE: Tree Drafting + Feature Reuse · Tree Attention Verification · When It Wins, When It Doesn't"
},
{
"name": "Gradient Checkpointing and Activation Recomputation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/10-llms-from-scratch/34-gradient-checkpointing/",
"summary": "Backprop keeps every intermediate activation. At 70B parameters and 128K context that is 3 TB of activations per rank. Checkpointing trades FLOPs for memory: recompute instead o…",
"keywords": "What Backward Actually Needs · Naive Full Checkpointing · Selective Checkpointing (Korthikanti 2022) · Offload · Recompute Cost Model · Memory Savings Model · When Not to Checkpoint · Implementation Patterns · Interaction with TP / PP / FP8 · Step 1: A Toy Model With Segments · Step 2: Naive Backward Needing All Activations · Step 3: Checkpoint-Every-k Memory · Step 4: Cost Model · Step 5: Memory Estimator · Step 6: Optimal Segment Size · Step 7: Selective Checkpoint Decision"
}
]
},
{
"id": 11,
"name": "LLM Engineering",
"status": "complete",
"desc": "Put LLMs to work in production.",
"lessons": [
{
"name": "Prompt Engineering: Techniques & Patterns",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/01-prompt-engineering/",
"summary": "Most people write prompts like they are texting a friend. Then they wonder why a 200-billion parameter model gives mediocre answers. Prompt engineering is not about tricks. It i…",
"keywords": "Anatomy of a Prompt · Role Prompting: Why \"You are an expert X\" Works · Instruction Clarity: Specific Beats Vague · Output Format Control · Constraint Specification · Temperature and Sampling · Context Windows: What Fits Where · Prompt Patterns · Anti-Patterns · Cross-Model Prompt Design · Step 1: Prompt Template Library · Step 2: Prompt Builder · Step 3: Multi-Model Testing Harness · Step 4: Prompt Comparison and Scoring · Step 5: Test Suite Runner · Step 6: Run Everything · OpenAI: Temperature and System Messages · Anthropic: System Message + Assistant Prefill · Google: Gemini with Safety Settings · LangChain: Provider-Agnostic Prompts"
},
{
"name": "Few-Shot, CoT, Tree-of-Thought",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/02-few-shot-cot/",
"summary": "Telling a model what to do is prompting. Showing it how to think is engineering. The gap between 78% and 91% accuracy on the same model, same task, same data is not a better mod…",
"keywords": "Zero-Shot vs Few-Shot: When Examples Beat Instructions · Example Selection: Similar Beats Random · Chain-of-Thought: Giving Models Scratch Paper · Self-Consistency: Sample Many, Vote Once · Tree-of-Thought: Branching Exploration · ReAct: Thinking + Doing · Structured Prompting: XML Tags, Delimiters, Headers · Prompt Chaining: Sequential Decomposition · Performance Comparison · Step 1: Few-Shot Example Store · Step 2: Chain-of-Thought Prompt Builder · Step 3: Self-Consistency Voting · Step 4: Tree-of-Thought Solver · Step 5: Full Pipeline · With LangChain · With DSPy · Comparison: From-Scratch vs Frameworks"
},
{
"name": "Structured Outputs",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/03-structured-outputs/",
"summary": "Your LLM returns a string. Your application needs JSON. That gap has crashed more production systems than any model hallucination. Structured output is the bridge between natura…",
"keywords": "The Structured Output Spectrum · JSON Schema: The Contract Language · The Pydantic Pattern · Function Calling / Tool Use · Common Failure Modes · Step 1: JSON Schema Validator · Step 2: Pydantic-Style Model to Schema · Step 3: Constrained Token Filter · Step 4: Extraction Pipeline · Step 5: Run the Full Pipeline · OpenAI Structured Outputs · Anthropic Tool Use · Instructor Library"
},
{
"name": "Embeddings & Vector Representations",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/04-embeddings/",
"summary": "Text is discrete. Math is continuous. Every time you ask an LLM to find \"similar\" documents, compare meanings, or search beyond keywords, you're relying on a bridge between thes…",
"keywords": "What Is an Embedding? · The Word2Vec Breakthrough · From Words to Sentences · Modern Embedding Models · Similarity Metrics · Vector Databases and HNSW · Chunking Strategies · Bi-Encoders vs Cross-Encoders · Matryoshka Embeddings · Binary Quantization · Step 1: Text Chunking · Step 2: Building Embeddings from Scratch · Step 3: Similarity Functions · Step 4: Vector Index with Brute-Force Search · Step 5: The Semantic Search Engine · Step 6: Comparing Similarity Metrics"
},
{
"name": "Context Engineering",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/05-context-engineering/",
"summary": "Prompt engineering is a subset. Context engineering is the whole game. A prompt is a string you type. Context is everything that goes into the model's window: system instruction…",
"keywords": "The Context Window is a Scarce Resource · Lost-in-the-Middle · Context Components · Context Compression Strategies · Memory Systems · Dynamic Context Assembly · Step 1: Token Counter · Step 2: Context Budget Manager · Step 3: Lost-in-the-Middle Reordering · Step 4: Conversation History Compressor · Step 5: Dynamic Tool Selector · Step 6: Full Context Assembly Pipeline · Claude Code's Context Strategy · Cursor's Dynamic Context Loading · ChatGPT Memory · RAG as Context Engineering"
},
{
"name": "RAG: Retrieval-Augmented Generation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/06-rag/",
"summary": "Your LLM knows everything up to its training cutoff. It knows nothing about your company's docs, your codebase, or last week's meeting notes. RAG solves this by retrieving relev…",
"keywords": "The RAG Pattern · Why RAG Beats Fine-Tuning · Embedding Models · Vector Similarity · Chunking Strategies · Vector Databases · The Full Pipeline · Real Numbers · Step 1: Document Chunking · Step 2: TF-IDF Embeddings · Step 3: Cosine Similarity Search · Step 4: Prompt Construction · Step 5: The Complete RAG Pipeline · Step 6: Generation (simulated)"
},
{
"name": "Advanced RAG: Chunking, Reranking",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/07-advanced-rag/",
"summary": "Basic RAG retrieves the top-k most similar chunks. That works for simple questions. It falls apart for multi-hop reasoning, ambiguous queries, and large corpora. Advanced RAG is…",
"keywords": "Hybrid Search: Semantic + Keyword · Reciprocal Rank Fusion (RRF) · Reranking · Query Transformation · Parent-Child Chunking · Metadata Filtering · Evaluation · Step 1: BM25 Implementation · Step 2: Reciprocal Rank Fusion · Step 3: Hybrid Search Pipeline · Step 4: Simple Reranker · Step 5: HyDE (Hypothetical Document Embeddings) · Step 6: Parent-Child Chunking · Step 7: Faithfulness Evaluation"
},
{
"name": "Fine-Tuning with LoRA & QLoRA",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/08-fine-tuning-lora/",
"summary": "Full fine-tuning a 7B model requires 56GB of VRAM. You don't have that. Neither do most companies. LoRA lets you fine-tune the same model in 6GB by training less than 1% of the …",
"keywords": "LoRA: Low-Rank Adaptation · The Scaling Factor: Alpha · Where to Apply LoRA · Rank Selection · QLoRA: 4-Bit Quantization + LoRA · The Quality Question · Real-World Costs · The 2026 PEFT stack · Merging Adapters · When NOT to Fine-Tune · Step 1: The LoRA Layer · Step 2: LoRA-Wrapped Linear Layer · Step 3: Inject LoRA into a Model · Step 4: Count Parameters · Step 5: Merge Weights Back · Step 6: Simulated QLoRA Quantization · Step 7: Training Loop · Step 8: Full Demo"
},
{
"name": "Function Calling & Tool Use",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/09-function-calling/",
"summary": "LLMs cannot do anything. They generate text. That is the entire capability. They cannot check the weather, query a database, send an email, run code, or read a file. Every \"AI a…",
"keywords": "The Function Calling Loop · Tool Definitions: The JSON Schema Contract · Provider Comparison · Tool Choice: Auto, Required, Specific · Parallel Function Calling · Structured Outputs vs Function Calling · Security: The Non-Negotiable Rules · Error Handling · MCP: Model Context Protocol · Step 1: Define the Tool Registry · Step 2: Implement 5 Tools · Step 3: Register All Tools · Step 4: Build the Function Calling Loop · Step 5: Argument Validation · Step 6: Run the Demo · OpenAI Function Calling · Anthropic Tool Use · MCP Integration"
},
{
"name": "Evaluation & Testing",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/10-evaluation/",
"summary": "You would never deploy a web app without tests. You would never ship a database migration without a rollback plan. But right now, most teams ship LLM applications by reading 10 …",
"keywords": "The Eval Taxonomy · LLM-as-Judge: The Workhorse · Rubric Design · The Eval Pipeline · Eval Datasets: The Foundation · Sample Size and Confidence · Regression Testing · Cost of Evals · Anti-Patterns · Real Tools · Step 1: Define the Eval Data Structures · Step 2: Build the LLM-as-Judge Scorer · Step 3: Build Automated Metrics · Step 4: Build the Confidence Interval Calculator · Step 5: Build the Eval Runner and Comparison Report · Step 6: Run the Demo · promptfoo Integration · DeepEval Integration · CI/CD Integration Pattern"
},
{
"name": "Caching, Rate Limiting & Cost",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/11-caching-cost/",
"summary": "Most AI startups do not die from bad models. They die from bad unit economics. A single GPT-4o call costs fractions of a cent. Ten thousand users making ten calls per day costs …",
"keywords": "The Cost Anatomy of an LLM Call · Provider Caching: Built-in Discounts · Semantic Caching: Your Custom Layer · Exact Caching: Hash and Match · Rate Limiting: Protecting Your Budget · Model Routing: Right Model for the Right Job · Cost Tracking: Know Where the Money Goes · Batching: Bulk Discounts · Budget Alerts and Circuit Breakers · The Optimization Stack · Real Savings: Before and After · Step 1: Cost Calculator · Step 2: Exact Cache · Step 3: Semantic Cache · Step 4: Rate Limiter · Step 5: Cost Tracker · Step 6: Model Router · Step 7: Run the Demo · Anthropic Prompt Caching · OpenAI Automatic Caching · OpenAI Batch API · Production Semantic Cache with Redis"
},
{
"name": "Guardrails & Safety",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/12-guardrails/",
"summary": "Your LLM application will be attacked. Not might. Will. The first prompt injection attempt against your production system will come within 48 hours of launch. The question is no…",
"keywords": "The Guardrail Sandwich · Attack Taxonomy · Input Guardrails · Output Guardrails · The Content Filtering Stack · Tools of the Trade · Defense-in-Depth · Real Attack Case Studies · The Honest Truth · Step 1: Input Guardrails · Step 2: Output Guardrails · Step 3: The Guardrail Pipeline · Step 4: Monitoring Dashboard · Step 5: Run the Demo · OpenAI Moderation API · LlamaGuard · NeMo Guardrails · Guardrails AI"
},
{
"name": "Building a Production LLM App",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/13-production-app/",
"summary": "You have built prompts, embeddings, RAG pipelines, function calling, caching layers, and guardrails. Separately. In isolation. Like practicing guitar scales without ever playing…",
"keywords": "Production Architecture · The Stack · Streaming: Why It Matters · Error Handling: The Three Layers · Observability: What to Measure · A/B Testing Prompts in Production · Real Architecture Examples · Scaling · Cost Projection · The Deployment Checklist · Step 1: Core Infrastructure · Step 2: Prompt Management · Step 3: Semantic Cache · Step 4: Guardrails · Step 5: LLM Caller with Retry and Streaming · Step 6: The Request Pipeline · Step 7: Run the Full Demo · FastAPI Server (Production Deployment) · Real API Integration · Docker Deployment"
},
{
"name": "Model Context Protocol (MCP)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/14-model-context-protocol/",
"summary": "Every LLM app built before 2025 invented its own tool schema. Then Anthropic shipped MCP, Claude adopted it, OpenAI adopted it, and by 2026 it is the default wire format for con…",
"keywords": "The handshake · What MCP is not · Step 1: a minimal MCP server · Step 2: calling an MCP server from a host · Step 3: streamable HTTP transport · Step 4: scoping and safety"
},
{
"name": "Prompt Caching & Context Caching",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/15-prompt-caching/",
"summary": "Your system prompt is 4,000 tokens. Your RAG context is 20,000 tokens. You send both with every request. You also pay for both — every time. Prompt caching lets the provider kee…",
"keywords": "The cache-friendly layout · The break-even calculation · Step 1: Anthropic prompt caching with explicit markers · Step 2: one-hour extended TTL · Step 3: OpenAI automatic caching · Step 4: Gemini explicit context caching · Step 5: measuring hit rate in production"
},
{
"name": "LangGraph: State Machines for Agents",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/16-langgraph-state-machines/",
"summary": "A ReAct loop written by hand is a `while True`. A ReAct loop written in LangGraph is a graph you can checkpoint, interrupt, branch, and time-travel through. The agent hasn't cha…",
"keywords": "The four superpowers · Reducers are the point · The ReAct graph in four nodes · StateGraph vs Send (fanout) · Subgraphs · Step 1: state and nodes · Step 2: run with a thread · Step 3: add a human-in-the-loop interrupt · Step 4: time-travel for debugging · Step 5: swap the checkpointer for production"
},
{
"name": "Agent Framework Tradeoffs",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/11-llm-engineering/17-agent-framework-tradeoffs/",
"summary": "Every framework sells the same demo (research agent builds a report) and hides the same bug (state schema fights with the orchestration layer). Pick the framework whose abstract…",
"keywords": "What \"abstraction\" actually means · The state question · The branching question · The observability question · Cost and latency · Interoperability"
}
]
},
{
"id": 12,
"name": "Multimodal AI",
"status": "complete",
"desc": "See, hear, read, and reason across modalities — from ViT patches to computer-use agents.",
"lessons": [
{
"name": "Vision Transformers and the Patch-Token Primitive",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/01-vision-transformer-patch-tokens/",
"summary": "Before anything multimodal, an image has to become a sequence of tokens a transformer can eat. The 2020 ViT paper answered this with 16x16 pixel patches, a linear projection, an…",
"keywords": "Patches as tokens · Positional embeddings · CLS token, pooled output, and register tokens · Pretraining: supervised, contrastive, masked, self-distilled · Scaling laws · Parameter count for a ViT · 2026 production config"
},
{
"name": "CLIP and Contrastive Vision-Language Pretraining",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/02-clip-contrastive-pretraining/",
"summary": "OpenAI's CLIP (2021) proved a single idea big enough to power the next five years: align an image encoder and a text encoder in the same vector space using only noisy web image-…",
"keywords": "The dual encoder · InfoNCE loss · Temperature · Why sigmoid scales better (SigLIP) · Zero-shot classification · Linear probes and finetuning · SigLIP 2: NaFlex and dense features · ALIGN, BASIC, OpenCLIP, EVA-CLIP · The zero-shot ceiling"
},
{
"name": "BLIP-2 Q-Former as Modality Bridge",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/03-blip2-qformer-bridge/",
"summary": "CLIP aligns image and text but cannot generate captions, answer questions, or hold a conversation. BLIP-2 (Salesforce, 2023) solved that with a small trainable bridge: 32 learna…",
"keywords": "Learnable queries · Architecture · Two-stage training · Parameter economics · InstructBLIP and the instruction-aware Q-Former · MiniGPT-4 and the projector-only approach · Why LLaVA went simpler · Gated cross-attention: Flamingo, the ancestor · The 2026 descendants"
},
{
"name": "Flamingo and Gated Cross-Attention",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/04-flamingo-gated-cross-attention/",
"summary": "DeepMind's Flamingo (2022) did two things before anyone else. It showed a single model could process arbitrarily interleaved sequences of images, videos, and text. And it showed…",
"keywords": "The frozen LLM · Perceiver resampler · Gated cross-attention · Masked cross-attention for interleaved inputs · In-context few-shot learning · Training data · OpenFlamingo and Otter · The descendants · Comparison to BLIP-2"
},
{
"name": "LLaVA and Visual Instruction Tuning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/05-llava-visual-instruction-tuning/",
"summary": "LLaVA (April 2023) is the most copied multimodal architecture on the planet. It replaced BLIP-2's Q-Former with a 2-layer MLP, replaced Flamingo's gated cross-attention with nai…",
"keywords": "The architecture · Stage 1: projector alignment · Stage 2: visual instruction tuning · Why the community copied this · LLaVA-1.5 and LLaVA-NeXT · LLaVA-OneVision · The comparison to Q-Former · The prompt format · Parameter economy"
},
{
"name": "Any-Resolution Vision — Patch-n'-Pack and NaFlex",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/06-any-resolution-patch-n-pack/",
"summary": "Real images are not 224x224 squares. A receipt is 9:16, a chart is 16:9, a medical scan might be 4096x4096, a mobile screenshot is 9:19.5. The pre-2024 VLM answer — resize every…",
"keywords": "NaViT and patch-n'-pack · AnyRes (LLaVA-NeXT) · M-RoPE (Qwen2-VL) · NaFlex (SigLIP 2) · The packing mask · Token budgets"
},
{
"name": "Open-Weight VLM Recipes: What Actually Matters",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/07-open-weight-vlm-recipes/",
"summary": "The 2024-2026 open-weight VLM literature is a forest of ablation tables. Apple's MM1 tested 13 combinations of image encoder, connector, and data mix. Allen AI's Molmo proved de…",
"keywords": "The five-axis design space · Axis 1: encoder > connector · Axis 2: connector design is a wash · Axis 3: LLM size sets the ceiling · Axis 4: data — detailed human captions beat distillation · Axis 5: resolution and its schedule · The Prismatic controlled comparison · A picker for 2026"
},
{
"name": "LLaVA-OneVision: Single, Multi, Video",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/08-llava-onevision-single-multi-video/",
"summary": "Before LLaVA-OneVision (Li et al., August 2024) the open-VLM world had separate lineages: LLaVA-1.5 for single images, multi-image models like Mantis and VILA, video models like…",
"keywords": "The OneVision token budget · The three-stage curriculum · Why curriculum works · Emergent cross-scenario skills · Visual-token pooling · LLaVA-OneVision-1.5 · Contrast with Qwen2.5-VL"
},
{
"name": "Qwen-VL Family and Dynamic-FPS Video",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/09-qwen-vl-family-dynamic-fps/",
"summary": "The Qwen-VL family — Qwen-VL (2023), Qwen2-VL (2024), Qwen2.5-VL (2025), Qwen3-VL (2025) — is the most influential open vision-language model lineage in 2026. Each generation ma…",
"keywords": "Qwen-VL (August 2023) · Qwen2-VL (September 2024) — M-RoPE and native resolution · Qwen2.5-VL (February 2025) — dynamic FPS + absolute time · Qwen3-VL (November 2025) · M-RoPE mathematically · Dynamic-FPS sampling logic · Structured agent output"
},
{
"name": "InternVL3 Native Multimodal Pretraining",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/10-internvl3-native-multimodal/",
"summary": "Every open VLM before InternVL3 followed the same three-step recipe: take a text LLM trained on trillions of text tokens, bolt on a vision encoder, then fine-tune the seams. Thi…",
"keywords": "Native multimodal pretraining · V2PE (variable visual position encoding) · Visual Resolution Router (ViR) · Decoupled Vision-Language deployment (DvD) · Single-stage vs multi-stage quality · InternVL3.5 and InternVL-U · Trade-offs of native pretraining"
},
{
"name": "Chameleon Early-Fusion Token-Only",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/11-chameleon-early-fusion-tokens/",
"summary": "Every VLM we have seen so far keeps images and text separate. Visual tokens come from a vision encoder, flow into a projector, then meet text inside the LLM. The vision and text…",
"keywords": "VQ-VAE as image tokenizer · The shared vocabulary · Mixed-modality generation · Training stability — QK-Norm, dropout, LayerNorm ordering · The tokenizer's reconstruction ceiling · Chameleon vs BLIP-2 / LLaVA · Fuyu and AnyGPT"
},
{
"name": "Emu3 Next-Token Prediction for Generation",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/12-emu3-next-token-for-generation/",
"summary": "BAAI's Emu3 (Wang et al., September 2024) is the 2024 result that should have ended the diffusion-versus-autoregressive debate. A single Llama-style decoder-only transformer, tr…",
"keywords": "The Emu3 tokenizer · Single-loss training · Classifier-free guidance and temperature · Three roles, one model · Benchmarks · Compute cost · Why it matters"
},
{
"name": "Transfusion Autoregressive + Diffusion",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/13-transfusion-autoregressive-diffusion/",
"summary": "Chameleon and Emu3 bet everything on discrete tokens. They work, but the quantization bottleneck is visible — the image quality plateaus below continuous-space diffusion models.…",
"keywords": "The two-loss architecture · Attention mask: causal text + bidirectional image · Diffusion loss inside the transformer · MMDiT: Stable Diffusion 3's variant · Why this beats Chameleon-style · What sits downstream"
},
{
"name": "Show-o Discrete-Diffusion Unified",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/14-show-o-discrete-diffusion-unified/",
"summary": "Transfusion mixes continuous and discrete representations. Show-o (Xie et al., August 2024) goes the other way: text tokens use causal next-token prediction, image tokens use ma…",
"keywords": "Masked discrete diffusion (MaskGIT) · Show-o: one transformer, hybrid mask · Parallel sampling · Tasks in one checkpoint · Masking schedule · Show-o2 · Where Show-o sits"
},
{
"name": "Janus-Pro Decoupled Encoders",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/15-janus-pro-decoupled-encoders/",
"summary": "Unified multimodal models have an unavoidable tension. Understanding wants semantic features — SigLIP or DINOv2 output vectors rich with concept-level information. Generation wa…",
"keywords": "Decoupled visual encoding · Why this works · Data scaling — Janus vs Janus-Pro · JanusFlow — the rectified flow variant · The shared body's job · Compared to InternVL-U · Limitations"
},
{
"name": "MIO Any-to-Any Streaming",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/16-mio-any-to-any-streaming/",
"summary": "GPT-4o ships a product most open models cannot replicate: an agent that hears voice, sees video, and speaks back in real time. The open-ecosystem answer by late 2024 was MIO (Wa…",
"keywords": "Four tokenizers for four modalities · Streaming decode · Four-stage curriculum · Chain-of-visual-thought · Competitors in any-to-any · Latency budget · Why any-to-any stays hard"
},
{
"name": "Video-Language Temporal Grounding",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/17-video-language-temporal-grounding/",
"summary": "Video is not a stack of photos. A 5-second clip has causal ordering, action verbs, and event timing that an image model cannot represent. Video-LLaMA (Zhang et al., June 2023) s…",
"keywords": "Video-LLaMA: Q-former per clip + audio branch · VideoChat and Video-LLaVA · Qwen2.5-VL and TMRoPE · Frame sampling strategies · Pooling per frame · The four video benchmarks · Grounding output formats · 2026 best practice"
},
{
"name": "Long-Video at Million-Token Context",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/18-long-video-million-token/",
"summary": "A 1-hour 4K video at 24 FPS, patched and embedded, produces on the order of 60 million tokens. A 2-hour podcast episode transcribed is 30,000 tokens. A full Blu-ray feature film…",
"keywords": "Path 1: Brute context (Gemini 1.5, Claude Opus) · Path 2: Ring attention (LWM, LongVILA) · Path 3: Token compression (Video-XL, LongVA) · Path 4: Agentic retrieval (VideoAgent) · Needle-in-a-haystack benchmarks · Which path to pick · 2026 production pattern"
},
{
"name": "Audio-Language Models: Whisper to AF3",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/19-audio-language-whisper-to-af3/",
"summary": "Whisper (Radford et al., December 2022) settled speech recognition — 680k hours of weakly-supervised multilingual speech, a simple encoder-decoder transformer, a benchmark that …",
"keywords": "Log-Mel spectrogram: the input feature · Whisper's encoder · BEATs and audio-specific encoders · Audio Q-former · The arc — SALMONN, Qwen-Audio, AF3 · Cascaded vs end-to-end · 2026 production recipe · MMAU — the audio reasoning benchmark"
},
{
"name": "Omni Models: Thinker-Talker Streaming",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/20-omni-models-thinker-talker/",
"summary": "GPT-4o's product demo in May 2024 was disruptive not because of the underlying model but because of the product shape — a voice interface where you talk, the model sees what the…",
"keywords": "Thinker and Talker · TMRoPE — time-aligned multimodal positions · Streaming speech synthesis · VAD and turn-taking · Qwen3-Omni (November 2025) · Production latency budget · Token-rate math"
},
{
"name": "Embodied VLAs: RT-2, OpenVLA, π0, GR00T",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/21-embodied-vlas-openvla-pi0-groot/",
"summary": "The first time a model read a recipe off a website and executed it in a kitchen robot was RT-2 (Google DeepMind, July 2023). RT-2 discretized actions as text tokens, co-fine-tun…",
"keywords": "Action tokenization (RT-2) · OpenVLA — the open 7B reference · FAST tokenizer — faster action decode · π0 and flow-matching actions · GR00T N1 — dual-system for humanoids · Open X-Embodiment · Co-fine-tuning vs robot-only · Safety and action limits"
},
{
"name": "Document and Diagram Understanding",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/22-document-diagram-understanding/",
"summary": "Documents are not photos. A PDF, scientific paper, invoice, or handwritten form has layout, tables, diagrams, footnotes, headers, and semantic structure that plain image underst…",
"keywords": "Era 1 — OCR pipeline (pre-2021) · TrOCR (2021) · Era 2 — OCR-free (2022-2023) · LayoutLMv3 (2022) · DocLLM (2023) · Era 3 — VLM-native (2024+) · The Claude 4.7 / GPT-5 frontier · Math equations and LaTeX output · Handwriting · 2026 recipe"
},
{
"name": "ColPali Vision-Native Document RAG",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/23-colpali-vision-native-rag/",
"summary": "Traditional RAG parses PDFs into text, splits into chunks, embeds chunks, stores vectors. Every step loses signal: OCR drops chart data, chunking breaks table rows, text embeddi…",
"keywords": "ColBERT (2020) · ColPali · ColQwen2 and ColSmol · VisRAG · M3DocRAG · ViDoRe — the benchmark · The end-to-end RAG pipeline · Storage math · When text-RAG still wins"
},
{
"name": "Multimodal RAG and Cross-Modal Retrieval",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/24-multimodal-rag-cross-modal/",
"summary": "Vision-native document RAG is one slice. Production multimodal RAG goes wider — retrieving across text, images, audio, and video for workflows like trip planning (\"find me a qui…",
"keywords": "Cross-modal retrieval · Fusion strategies · Generation grounding · The 2025 surveys · MuRAG — the foundational paper · A production trip-planner example · Agentic multimodal RAG · Evaluation"
},
{
"name": "Multimodal Agents and Computer-Use (Capstone)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/12-multimodal-ai/25-multimodal-agents-computer-use/",
"summary": "The 2026 frontier product is a multimodal agent that reads screenshots, clicks buttons, navigates web UIs, fills forms, and completes workflows end-to-end. SeeClick and CogAgent…",
"keywords": "GUI grounding — the primitive · Action schemas · Screenshot-only vs accessibility-tree · Long-horizon memory · Visual tool use · The 2026 benchmarks · Why it's still hard · The capstone build-it"
}
]
},
{
"id": 13,
"name": "Tools & Protocols",
"status": "complete",
"desc": "The interfaces between AI and the real world.",
"lessons": [
{
"name": "The Tool Interface",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/01-the-tool-interface/",
"summary": "A language model produces tokens. A program takes actions. The gap between those two is the tool interface: a contract that lets the model request an action and the host execute…",
"keywords": "Step one: describe · Step two: decide · Step three: execute · Step four: observe · The trust split · Where the loop lives · Why not just prompt the model to emit JSON? · Circuit breakers · Where Phase 13 goes from here"
},
{
"name": "Function Calling Deep Dive",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/02-function-calling-deep-dive/",
"summary": "The three frontier providers converged on the same tool-call loop in 2024 and then diverged on everything else. OpenAI uses `tools` and `tool_calls`. Anthropic uses `tool_use` a…",
"keywords": "The common structure · Shape diffs, field by field · Limits you will actually hit · `tool_choice` behavior · Parallel calls · Streaming · Errors and repair · The translator pattern"
},
{
"name": "Parallel and Streaming Tool Calls",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/03-parallel-and-streaming-tool-calls/",
"summary": "Three independent weather lookups serialized is three round trips. Run them in parallel and total time collapses to the slowest single call. Every frontier provider now emits mu…",
"keywords": "Enabling parallel · Id correlation · Running calls concurrently · Streaming tool calls · Partial JSON and the parse-early trap · Out-of-order completion · Benchmark: sequential vs parallel · Streaming fan-out wall-clock"
},
{
"name": "Structured Output",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/04-structured-output/",
"summary": "\"Ask the model nicely to return JSON\" fails 5 to 15 percent of the time, even on frontier models. Structured outputs close that gap with constrained decoding: the model is liter…",
"keywords": "JSON Schema 2020-12 — the lingua franca · Pydantic, the Python binding · Zod, the TypeScript binding · Refusals · Constrained decoding in the open · The three failure modes · Retry strategy · Small-model support"
},
{
"name": "Tool Schema Design",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/05-tool-schema-design/",
"summary": "A correct tool fails silently when the model cannot tell when to use it. Naming, descriptions, and parameter shapes drive 10 to 20 percentage-point swings in tool-selection accu…",
"keywords": "Naming rules · Description pattern · Atomic vs monolithic · Parameter design · Error messages as teaching signals · Versioning · Tool poisoning prevention · Benchmarks"
},
{
"name": "MCP Fundamentals",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/06-mcp-fundamentals/",
"summary": "Every integration before MCP was a one-off. The Model Context Protocol, first shipped by Anthropic in November 2024 and now stewarded by the Linux Foundation's Agentic AI Founda…",
"keywords": "Three server primitives · Three client primitives · Wire format: JSON-RPC 2.0 · Three-phase lifecycle · Capability negotiation · Structured content and error shapes · Client capabilities vs tool call details · Why JSON-RPC and not REST?"
},
{
"name": "Building an MCP Server",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/07-building-an-mcp-server/",
"summary": "Most MCP tutorials show only stdio hello-worlds. A real server exposes tools plus resources plus prompts, handles capability negotiation, emits structured errors, and works the …",
"keywords": "Dispatch loop · Implementing `initialize` · Implementing `tools/list` and `tools/call` · Implementing resources · Implementing prompts · Stdio transport subtleties · Annotations · Graduation path"
},
{
"name": "Building an MCP Client",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/08-building-an-mcp-client/",
"summary": "Most MCP content ships server tutorials and waves a hand at the client. Client code is where the hard orchestration lives: process spawning, capability negotiation, tool list me…",
"keywords": "Child-process spawning · Per-server session state · Merged namespace · Routing · Sampling callback · Notification handling · Reconnection · Keepalive and session id"
},
{
"name": "MCP Transports",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/09-mcp-transports/",
"summary": "stdio works locally and nowhere else. Streamable HTTP (2025-03-26) is the remote standard. The old HTTP+SSE transport is deprecated and being removed in mid-2026. Picking the wr…",
"keywords": "stdio · Streamable HTTP · Single endpoint vs two · `Origin` validation and DNS-rebinding · Session id lifecycle · Keepalive and reconnect · Backwards compatibility probe · Cloudflare, ngrok, and hosting · Gateway composition · Transport failure modes · When to bypass Streamable HTTP"
},
{
"name": "MCP Resources and Prompts",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/10-mcp-resources-and-prompts/",
"summary": "Tools get 90 percent of MCP attention. The other two server primitives solve different problems. Resources expose data for reading; prompts expose reusable templates as slash-co…",
"keywords": "Tools vs resources vs prompts — the decision rule · Resources · Resource subscriptions · Resource templates (2025-11-25 addition) · Prompts · Hosts and prompts · The \"list changed\" notification · Content type conventions · Dynamic resources · Subscriptions vs polling · Prompts vs system prompts"
},
{
"name": "MCP Sampling",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/11-mcp-sampling/",
"summary": "Most MCP servers are dumb executors: take arguments, run code, return content. Sampling lets a server flip direction: it asks the client's LLM to make a decision. This enables s…",
"keywords": "`sampling/createMessage` request · `modelPreferences` · `includeContext` · Sampling with tools (SEP-1577) · Human-in-the-loop · Server-hosted loops without API keys · Safety risks (Unit 42 disclosure, 2026 Q1)"
},
{
"name": "MCP Roots and Elicitation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/12-mcp-roots-and-elicitation/",
"summary": "Hard-coded paths break the moment a user opens a different project. Pre-filled tool arguments break when the user under-specifies. Roots scope the server to a user-controlled se…",
"keywords": "Roots · Why roots are a client primitive · Elicitation: the form-mode default · Elicitation: URL mode (SEP-1036, experimental) · When elicitation is the right tool · When elicitation is wrong · Human-in-the-loop bridge"
},
{
"name": "MCP Async Tasks",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/13-mcp-async-tasks/",
"summary": "Real agent work takes minutes to hours: CI runs, deep-research synthesis, batch exports. Synchronous tool calls drop connections, time out, or block the UI. SEP-1686, merged in …",
"keywords": "Task augmentation · Per-tool opt-in · States · Methods · Streaming state changes · Durable state · Cancellation semantics · Crash recovery · Async tasks plus sampling · Why this is experimental"
},
{
"name": "MCP Apps",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/14-mcp-apps/",
"summary": "Text-only tool output caps what agents can show. MCP Apps (SEP-1724, official January 26, 2026) let a tool return sandboxed interactive HTML rendered inline in Claude Desktop, C…",
"keywords": "The `ui://` resource scheme · Iframe sandbox · postMessage protocol · Permissions · Security risks · `ui/initialize` handshake · AppRenderer / AppFrame SDK primitives · Ecosystem status"
},
{
"name": "MCP Security I — Tool Poisoning",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/15-mcp-security-tool-poisoning/",
"summary": "Tool descriptions land in the model's context verbatim. Malicious servers embed hidden instructions that users never see. Research in 2025-2026 from Invariant Labs, Unit 42, and…",
"keywords": "Attack 1: tool poisoning · Attack 2: rug pulls · Attack 3: cross-server tool shadowing · Attack 4: MCP Preference Manipulation Attacks (MPMA) · Attack 5: parasitic toolchains · Attack 6: sampling attacks · Attack 7: supply-chain masquerading · The Rule of Two (Meta, 2026) · Defenses that work · Defenses that do not work alone"
},
{
"name": "MCP Security II — OAuth 2.1",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/16-mcp-security-oauth-2-1/",
"summary": "Remote MCP servers need authorization, not just authentication. The 2025-11-25 spec aligns with OAuth 2.1 + PKCE + resource indicators (RFC 8707) + protected-resource metadata (…",
"keywords": "Roles · Authorization code + PKCE · Protected-resource metadata (RFC 9728) · Resource indicators (RFC 8707) · Scope model · Step-up authorization (SEP-835) · Token audience validation · Short-lived tokens and rotation · No token passthrough · Confused deputy prevention · Client ID discovery · Gateways and OAuth"
},
{
"name": "MCP Gateways and Registries",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/17-mcp-gateways-and-registries/",
"summary": "Enterprises cannot let every dev install random MCP servers. A gateway centralizes auth, RBAC, audit, rate limiting, caching, and tool-poisoning detection, then exposes the merg…",
"keywords": "Five gateway responsibilities · Gateway as a single endpoint · Credential vaulting · Tool-hash pinning at the gateway · Policy-as-code · Session-aware routing · Namespace merging · Registries · Reverse-DNS naming · Vendor survey, April 2026"
},
{
"name": "MCP Auth in Production — Enrollment, JWKS Refresh, Audience Pinning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/18-mcp-auth-production/",
"summary": "Lesson 16 stood up the OAuth 2.1 state machine in memory. By 2026, every MCP server you ship to a real org sits behind production auth: client enrollment that scales to an unbou…",
"keywords": "RFC 8414 — OAuth Authorization Server Metadata · RFC 9728 (recap) — Protected Resource Metadata · Client ID Metadata Documents (the recommended default) · RFC 7591 — Dynamic Client Registration (fallback / backwards compatibility) · RFC 8707 (recap) — Resource Indicators · RFC 7636 (recap) — PKCE · MCP Spec 2025-11-25 Auth Profile · IdP capability matrix · JWKS refresh pattern (rotate at the AS, refresh at the resource server) · The validation routine · Audience-replay walkthrough (access-token privilege restriction) · Mix-up attacks (a client-side defense the server cannot provide) · Failure modes"
},
{
"name": "A2A Protocol",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/19-a2a-protocol/",
"summary": "MCP is agent-to-tool. A2A (Agent2Agent) is agent-to-agent — an open protocol for letting opaque agents built on different frameworks collaborate. Released by Google in April 202…",
"keywords": "Agent Card · Signed Agent Cards (AP2) · Task lifecycle · Messages and Parts · Artifacts · Two transport bindings · Opacity preservation · Timeline · Relationship to MCP"
},
{
"name": "OpenTelemetry GenAI",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/20-opentelemetry-genai/",
"summary": "An agent calls five tools, three MCP servers, and two sub-agents. You need one trace across all of it. The OpenTelemetry GenAI semantic conventions (stable attributes in v1.37 a…",
"keywords": "Span hierarchy · Required attributes · Span kinds · Opt-in content capture · Events on spans · Exporters · Propagation across MCP · Metrics · AgentOps layer"
},
{
"name": "LLM Routing Layer",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/21-llm-routing-layer/",
"summary": "Provider lock-in is expensive. Different tool-calling workloads suit different models. Routing gateways give one API surface, retries, failover, cost tracking, and guardrails. T…",
"keywords": "OpenAI-compatible proxy shape · Model aliases · Fallback chains · Semantic caching · Guardrails · Per-key rate limits · Self-hosted vs managed trade-offs · Cost tracking · MCP plus routing · Routing strategies"
},
{
"name": "Skills and Agent SDKs",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/22-skills-and-agent-sdks/",
"summary": "MCP says \"what tools exist.\" Skills say \"how to do a task.\" The 2026 stack layers both. Anthropic's Agent Skills (open standard, December 2025) ship as SKILL.md with progressive…",
"keywords": "AGENTS.md (agents.md) · SKILL.md format · Progressive disclosure · Filesystem discovery · Anthropic Claude Agent SDK · OpenAI Apps SDK · Cross-agent portability via SkillKit · The three-layer stack"
},
{
"name": "Capstone — Tool Ecosystem",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/13-tools-and-protocols/23-capstone-tool-ecosystem/",
"summary": "Phase 13 taught every piece. This capstone wires them into one production-shaped system: an MCP server with tools + resources + prompts + tasks + UI, OAuth 2.1 at the edge, an R…",
"keywords": "Architecture · Trace hierarchy · Security posture · Rendering · Packaging · What each Phase 13 lesson contributed"
}
]
},
{
"id": 14,
"name": "Agent Engineering",
"status": "complete",
"desc": "Build agents from first principles — loop, memory, planning, frameworks, benchmarks, production, workbench.",
"lessons": [
{
"name": "The Agent Loop",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/01-the-agent-loop/",
"summary": "Every agent in 2026 — Claude Code, Cursor, Devin, Operator — is a variant of the ReAct loop from 2022. Reasoning tokens interleave with tool calls and observations until a stop …",
"keywords": "ReAct: the canonical format · The 2026 shift: native reasoning · The five ingredients · Why this loop is everywhere · 2026 pitfalls"
},
{
"name": "ReWOO and Plan-and-Execute",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/02-rewoo-plan-and-execute/",
"summary": "ReAct interleaves thought and action in one stream. ReWOO separates them: one big plan up front, then execute. 5x fewer tokens, +4% accuracy on HotpotQA, and you can distill the…",
"keywords": "The three roles · Why 5x fewer tokens · Why it is more robust · Planner distillation · Plan-and-Execute (LangChain, 2023) · Plan-and-Act (Erdogan et al., arXiv:2503.09572, ICML 2025) · When to pick which"
},
{
"name": "Reflexion and Verbal Reinforcement Learning",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/03-reflexion-verbal-rl/",
"summary": "Gradient-based RL needs thousands of trials and a GPU cluster to fix a failure mode. Reflexion (Shinn et al., NeurIPS 2023) does it in natural language: after each failed trial,…",
"keywords": "The three components · Three evaluator types · Why this generalizes · When it works and when it does not"
},
{
"name": "Tree of Thoughts and LATS",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/04-tree-of-thoughts-lats/",
"summary": "A single chain-of-thought trajectory has no room to backtrack. ToT (Yao et al., 2023) turns reasoning into a tree with self-evaluation on each node. LATS (Zhou et al., 2024) uni…",
"keywords": "Tree of Thoughts (Yao et al., NeurIPS 2023) · LATS (Zhou et al., ICML 2024) · MCTS, minimally · The cost reality · 2026 positioning"
},
{
"name": "Self-Refine and CRITIC",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/05-self-refine-and-critic/",
"summary": "Self-Refine (Madaan et al., 2023) uses one LLM in three roles — generate, feedback, refine — in a loop. Average gain: +20 absolute on 7 tasks. CRITIC (Gou et al., 2023) hardens …",
"keywords": "Self-Refine (Madaan et al., NeurIPS 2023) · CRITIC (Gou et al., arXiv:2305.11738, v4 Feb 2024) · The stop condition · Evaluator-Optimizer (Anthropic, 2024) · OpenAI Agents SDK output guardrails · 2026 pitfalls"
},
{
"name": "Tool Use and Function Calling",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/06-tool-use-and-function-calling/",
"summary": "Toolformer (Schick et al., 2023) started self-supervised tool annotation. Berkeley Function Calling Leaderboard V4 (Patil et al., 2025) sets the 2026 bar: 40% agentic, 30% multi…",
"keywords": "Toolformer (Schick et al., NeurIPS 2023) · Berkeley Function Calling Leaderboard V4 (Patil et al., ICML 2025) · Tool schema · Argument validation · Parallel tool calls · Sandboxing"
},
{
"name": "Memory — Virtual Context and MemGPT",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/07-memory-virtual-context-memgpt/",
"summary": "Context windows are finite. Conversations, documents, and tool traces are not. MemGPT (Packer et al., 2023) frames this as OS virtual memory — main context is RAM, external stor…",
"keywords": "MemGPT: the OS analogy · Two tiers · The interrupt pattern · Where MemGPT ends and Letta begins · Where this pattern goes wrong"
},
{
"name": "Memory Blocks and Sleep-Time Compute",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/08-memory-blocks-sleep-time-compute/",
"summary": "MemGPT became Letta in 2024. The 2026 evolution adds two ideas: discrete functional memory blocks the model can edit directly, and a sleep-time agent that consolidates memory as…",
"keywords": "Three tiers · Memory blocks · Sleep-time compute · Letta V1 and native reasoning · Where this pattern goes wrong"
},
{
"name": "Hybrid Memory — Mem0 Vector + Graph + KV",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/09-hybrid-memory-mem0/",
"summary": "Mem0 (Chhikara et al., 2025) treats memory as three stores in parallel — vector for semantic similarity, KV for fast fact lookup, graph for entity-relationship reasoning. A scor…",
"keywords": "Three stores in parallel · Fusion scoring · Mem0g and temporal reasoning · Benchmark numbers · Scope taxonomy · Where this pattern goes wrong"
},
{
"name": "Skill Libraries and Lifelong Learning — Voyager",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/10-skill-libraries-voyager/",
"summary": "Voyager (Wang et al., TMLR 2024) treats executable code as a skill. Skills are named, retrievable, composable, and refined by environment feedback. This is the reference archite…",
"keywords": "Three components · Action space = code · Skill retrieval · Iterative refinement · Curriculum and exploration · Where this pattern goes wrong"
},
{
"name": "Planning with HTN and Evolutionary Search",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/11-planning-htn-and-evolutionary/",
"summary": "Symbolic planning handles the cases where the plan is provably correct. Evolutionary code search handles the cases where the fitness function is machine-checkable. ChatHTN (2025…",
"keywords": "Hierarchical Task Networks · ChatHTN (Gopalakrishnan et al., 2025) · AlphaEvolve (Novikov et al., 2025) · When to use which · Where this pattern goes wrong"
},
{
"name": "Anthropic's Workflow Patterns",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/12-anthropic-workflow-patterns/",
"summary": "Schluntz and Zhang (Anthropic, Dec 2024) distinguish workflows (predefined paths) from agents (dynamic tool-use). Five workflow patterns cover most cases. Start with direct API …",
"keywords": "Workflows vs agents · The augmented LLM · The five patterns · Where workflows beat agents · Where agents beat workflows · The context-engineering companion"
},
{
"name": "LangGraph — Stateful Graphs and Durable Execution",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/13-langgraph-stateful-graphs/",
"summary": "LangGraph is the 2026 reference for low-level stateful orchestration. Agent is a state machine; nodes are functions; edges are transitions; state is immutable and checkpointed a…",
"keywords": "The graph · Durable execution · Streaming · Human-in-the-loop · Memory · Three topologies · Where this pattern goes wrong"
},
{
"name": "AutoGen v0.4 — Actor Model",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/14-autogen-actor-model/",
"summary": "AutoGen v0.4 (Microsoft Research, Jan 2025) redesigned agent orchestration around the actor model. Async message exchange, event-driven agents, fault isolation, natural concurre…",
"keywords": "Actors · Three API layers in AutoGen v0.4 · Why decoupling matters · Topologies · Observability · Status: maintenance mode"
},
{
"name": "CrewAI — Role-Based Crews and Flows",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/15-crewai-role-based-crews/",
"summary": "CrewAI is the 2026 role-based multi-agent framework. Four primitives: Agent, Task, Crew, Process. Two top-level shapes: Crews (autonomous, role-based collaboration) and Flows (e…",
"keywords": "Four primitives · Sequential vs Hierarchical vs Consensus · Crews vs Flows · Tool integration · Memory hooks · When CrewAI fits · When CrewAI does not fit · Dependency shape · Where this pattern goes wrong"
},
{
"name": "OpenAI Agents SDK — Handoffs, Guardrails, Tracing",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/16-openai-agents-sdk/",
"summary": "OpenAI Agents SDK is the lightweight multi-agent framework built on the Responses API. Five primitives: Agent, Handoff, Guardrail, Session, Tracing. Handoffs are tools named `tr…",
"keywords": "Five primitives · Handoffs as tools · Guardrails · Tracing · Sessions · Where this pattern goes wrong"
},
{
"name": "Claude Agent SDK — Subagents and Session Store",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/17-claude-agent-sdk/",
"summary": "The Claude Agent SDK is the library form of the Claude Code harness. Built-in tools, subagents for context isolation, hooks, W3C trace propagation, session store parity. Claude …",
"keywords": "Client SDK vs Agent SDK · Built-in tools · Subagents · Session store · Hooks · W3C trace context · Claude Managed Agents · Where this pattern goes wrong"
},
{
"name": "Agno and Mastra — Production Runtimes",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/18-agno-and-mastra-runtimes/",
"summary": "Agno (Python) and Mastra (TypeScript) are the 2026 production-runtime pairing. Agno aims at microsecond agent instantiation and stateless FastAPI backends. Mastra ships agents, …",
"keywords": "Agno · Mastra · Positioning · When to pick each · Where this pattern goes wrong"
},
{
"name": "Benchmarks — SWE-bench, GAIA, AgentBench",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/19-benchmarks-swebench-gaia/",
"summary": "Three benchmarks anchor agent evaluation in 2026. SWE-bench tests code patching. GAIA tests generalist tool use. AgentBench tests multi-environment reasoning. Know their composi…",
"keywords": "SWE-bench (Jimenez et al., ICLR 2024 oral) · SWE-bench Verified · Contamination · GAIA (Mialon et al., Nov 2023) · AgentBench (Liu et al., ICLR 2024) · What these do not measure · Where benchmarking goes wrong"
},
{
"name": "Benchmarks — WebArena and OSWorld",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/20-benchmarks-webarena-osworld/",
"summary": "WebArena tests web-agent capability across four self-hosted apps. OSWorld tests desktop-agent capability across Ubuntu, Windows, macOS. At release (2023–2024) both showed a big …",
"keywords": "WebArena (Zhou et al., ICLR 2024) · Extensions · OSWorld (Xie et al., NeurIPS 2024) · Primary failure modes · Follow-ups · Why this matters · Where benchmarking goes wrong"
},
{
"name": "Computer Use — Claude, OpenAI CUA, Gemini",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/21-computer-use-agents/",
"summary": "Three production computer-use models in 2026. All three are vision-based. All three treat screenshots, DOM text, and tool outputs as untrusted input. Only direct user instructio…",
"keywords": "Claude computer use (Anthropic, Oct 22 2024) · OpenAI CUA / Operator (Jan 2025) · Gemini 2.5 Computer Use (Google DeepMind, Oct 7 2025) · The shared contract: untrusted input · When to pick which · Where this pattern goes wrong"
},
{
"name": "Voice Agents — Pipecat and LiveKit",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/22-voice-agents-pipecat-livekit/",
"summary": "Voice agents are a first-class production category in 2026. Pipecat gives you a Python frame-based pipeline (VAD → STT → LLM → TTS → transport). LiveKit Agents bridges AI models…",
"keywords": "Pipecat (pipecat-ai/pipecat) · LiveKit Agents (livekit/agents) · Commercial platforms · Where this pattern goes wrong · Typical 2026 latencies"
},
{
"name": "OpenTelemetry GenAI Semantic Conventions",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/23-otel-genai-conventions/",
"summary": "OpenTelemetry's GenAI SIG (launched April 2024) defines the standard schema for agent telemetry. Span names, attributes, and content-capture rules converge across vendors so age…",
"keywords": "Span categories · Agent span naming · Key attributes · Content capture · Stability · Where this pattern goes wrong"
},
{
"name": "Agent Observability — Langfuse, Phoenix, Opik",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/24-agent-observability-platforms/",
"summary": "Three open-source agent observability platforms dominate 2026. Langfuse (MIT) — 6M+ installs/month, tracing + prompt management + evals + session replay. Arize Phoenix (Elastic …",
"keywords": "Langfuse (MIT) · Arize Phoenix (Elastic License 2.0) · Comet Opik (Apache 2.0) · Industry data · Picking one · Where this pattern goes wrong"
},
{
"name": "Multi-Agent Debate and Collaboration",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/25-multi-agent-debate/",
"summary": "Du et al. (ICML 2024, \"Society of Minds\") run N model instances that independently propose answers, then iteratively critique each other over R rounds to converge. Improves fact…",
"keywords": "Society of Minds (Du et al., ICML 2024) · Sparse topology · When debate helps · When debate hurts · 2026 practical instantiations · Where this pattern goes wrong"
},
{
"name": "Failure Modes — Why Agents Break",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/26-failure-modes-agentic/",
"summary": "MASFT (Berkeley, 2025) catalogs 14 multi-agent failure modes in 3 categories. Microsoft's Taxonomy documents how existing AI failures amplify in agentic settings. Industry field…",
"keywords": "MASFT (Berkeley, arXiv:2503.13657) · Microsoft Taxonomy of Failure Mode in Agentic AI Systems · Characterizing Faults in Agentic AI (arXiv:2603.06847) · LLM Agent Hallucinations Survey (arXiv:2509.18970) · The five industry-recurring modes · Mitigation: gates at every step · Where failure monitoring goes wrong"
},
{
"name": "Prompt Injection and the PVE Defense",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/27-prompt-injection-defense/",
"summary": "Greshake et al. (AISec 2023) established indirect prompt injection as the defining agent security problem. Attacker plants instructions in data the agent retrieves; on ingest, t…",
"keywords": "Greshake et al., AISec 2023 (arXiv:2302.12173) · The 2026 defense doctrine · PVE: Prompt-Validator-Executor · Where defenses fail"
},
{
"name": "Orchestration Patterns — Supervisor, Swarm, Hierarchical",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/28-orchestration-patterns/",
"summary": "Four orchestration patterns recur across 2026 frameworks: supervisor-worker, swarm / peer-to-peer, hierarchical, debate. Anthropic's guidance: \"It's about building the right sys…",
"keywords": "Supervisor-worker · Swarm / peer-to-peer · Hierarchical · Debate · CrewAI Crew vs Flow · Anthropic's guidance · Where this pattern goes wrong"
},
{
"name": "Production Runtimes — Queue, Event, Cron",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/29-production-runtimes/",
"summary": "Production agents run on six runtime shapes: request-response, streaming, durable execution, queue-based background, event-driven, and scheduled. Pick the shape before you pick …",
"keywords": "Request-response · Streaming · Durable execution · Queue-based / background · Event-driven · Scheduled · 2026 deployment patterns · Observability is load-bearing · Where production runtimes fail"
},
{
"name": "Eval-Driven Agent Development",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/30-eval-driven-agent-development/",
"summary": "Anthropic's guidance: \"start with simple prompts, optimize them with comprehensive evaluation, and add multi-step agentic systems only when needed.\" Evaluation is not the last s…",
"keywords": "Three evaluation layers · Evaluator-optimizer (Anthropic) · 2026 best practice · Tying Phase 14 together · Where eval-driven development fails"
},
{
"name": "Agent Workbench: Why Capable Models Still Fail",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/31-agent-workbench-why-models-fail/",
"summary": "A capable model is not enough. Reliable agents need a workbench: instructions, state, scope, feedback, verification, review, and handoff. Strip those away and even a frontier mo…",
"keywords": "Workbench versus prompt engineering · Workbench versus framework · Reasoning from primitives, not from vendor taxonomies · Patterns in circulation, translated to primitives · What the receipts actually say · Where vendor writeups stop short"
},
{
"name": "The Minimal Agent Workbench",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/32-minimal-agent-workbench/",
"summary": "The smallest useful workbench is three files: a root instructions router, a state file, and a task board. Everything else is layered on top. If a repo cannot carry these three, …",
"keywords": "AGENTS.md is a router, not a manual · agent_state.json is the system of record · task_board.json is the queue · Three files is the floor, not the ceiling"
},
{
"name": "Agent Instructions as Executable Constraints",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/33-instructions-as-executable-constraints/",
"summary": "Instructions written as prose are wishes. Instructions written as constraints are tests. The workbench turns each rule into something an agent can check at runtime and a reviewe…",
"keywords": "Five categories that cover most rules · Rules are machine-readable · Rules are diff-friendly · Rules versus framework guardrails · Progressive disclosure: a map, not an encyclopedia"
},
{
"name": "Repo Memory and Durable State",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/34-repo-memory-and-state/",
"summary": "Chat history is volatile. The repo is durable. The workbench stores agent state in versioned files so the next session, the next agent, and the next reviewer all read from the s…",
"keywords": "What belongs in repo memory · Schema-first state · Atomic writes · Migrations"
},
{
"name": "Initialization Scripts for Agents",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/35-initialization-scripts/",
"summary": "Every session that starts cold pays a tax. The agent reads the same files, retries the same probes, and rediscovers the same paths. An init script pays the tax once and writes t…",
"keywords": "What the init script probes · Fail loud, fail fast, fail in one place · Idempotent · Init versus startup rules"
},
{
"name": "Scope Contracts and Task Boundaries",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/36-scope-contracts/",
"summary": "The model does not know where the work ends. A scope contract is a per-task file that says where the work begins, where it ends, and how to roll back if it spills. The contract …",
"keywords": "What goes in a scope contract · Globs, not raw paths · Rollback is part of scope · Scope check is a diff check · Two altitudes of scope: the feature list and the task contract"
},
{
"name": "Runtime Feedback Loops",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/37-runtime-feedback-loops/",
"summary": "Agents that do not see real command output guess. A feedback runner captures stdout, stderr, exit code, and timing into a structured record the next turn can read. Then the agen…",
"keywords": "What goes in a feedback record · Truncation is deterministic · Feedback versus telemetry · Refuse to advance without feedback"
},
{
"name": "Verification Gates",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/38-verification-gates/",
"summary": "The agent does not get to mark its own work as done. A verification gate reads the scope contract, the feedback log, the rule report, and the diff, and answers a single question…",
"keywords": "What the gate checks · Deterministic, not probabilistic · One report, one path · Refuse without exception"
},
{
"name": "Reviewer Agent: Separate Builder from Marker",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/39-reviewer-agent/",
"summary": "The agent that wrote the code cannot grade it. A reviewer is a second loop with a different system prompt, a different goal, and read-only access to everything the builder produ…",
"keywords": "Reviewer rubric · The reviewer is a separate role, not a separate model · The reviewer cannot edit the diff · Reviewer rubric versus verification gate"
},
{
"name": "Multi-Session Handoff",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/40-multi-session-handoff/",
"summary": "The session is going to end. The work is not. The handoff packet is the artifact that turns \"the agent worked for an hour\" into \"the next session is productive in the first minu…",
"keywords": "Seven fields every handoff carries · Handoffs are generated, not written · Two forms: human-readable and machine-readable · Feedback log trimming · Leave a clean state"
},
{
"name": "The Workbench on a Real Repo",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/41-workbench-for-real-repos/",
"summary": "Eleven lessons of surfaces are worth nothing if they do not survive contact with a real codebase. This lesson runs the same task twice on a small sample app: prompt-only versus …",
"keywords": "The sample app · The task · The two pipelines · The five outcomes measured"
},
{
"name": "Capstone: Ship a Reusable Agent Workbench Pack",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/14-agent-engineering/42-agent-workbench-capstone/",
"summary": "The mini-track ends with a pack you drop into any repo. Eleven lessons of surfaces compressed into a directory you can `cp -r` and have an agent working reliably the next mornin…",
"keywords": "The pack layout · What stays in, what stays out · The installer · Versioning"
}
]
},
{
"id": 15,
"name": "Autonomous Systems",
"status": "complete",
"desc": "Long-horizon agents, self-improvement, and the 2026 safety stack.",
"lessons": [
{
"name": "From Chatbots to Long-Horizon Agents (METR)",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/01-long-horizon-agents/",
"summary": "In 2023 a chatbot answered a question in one turn. In 2026 a frontier model routinely runs minutes to hours on a single task. METR's Time Horizon 1.1 benchmark (January 2026) pu…",
"keywords": "The METR Time Horizon, in one paragraph · What actually breaks when the horizon grows · Doubling times and what they imply · Eval-context gaming · Single-turn vs long-horizon, compared"
},
{
"name": "STaR, V-STaR, Quiet-STaR: Self-Taught Reasoning",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/02-star-family-reasoning/",
"summary": "The smallest possible self-improvement loop sits inside the rationale. A model generates a chain of thought, keeps the ones that land on correct answers, and fine-tunes on those…",
"keywords": "STaR: bootstrap on what worked · V-STaR: train a verifier with DPO · Quiet-STaR: per-token internal rationales · Why all three share a safety concern · Comparison · Where this sits in the 2026 stack"
},
{
"name": "AlphaEvolve: Evolutionary Coding Agents",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/03-alphaevolve-evolutionary-coding/",
"summary": "Pair a frontier coding model with an evolutionary loop and a machine-checkable evaluator. Let the loop run long enough. It discovers a 4x4 complex-matrix multiplication procedur…",
"keywords": "The loop · What makes the evaluator non-negotiable · Reward hacking is the other face of that statement · Why LLM + search beats either alone · Where AlphaEvolve fits in the frontier stack"
},
{
"name": "Darwin Gödel Machine: Self-Modifying Agents",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/04-darwin-godel-machine/",
"summary": "Schmidhuber's 2003 Godel Machine required a formal proof that any self-modification was beneficial before accepting it. That proof is impossible in practice. Darwin Godel Machin…",
"keywords": "The loop · What DGM actually improved · The reward-hacking demonstration · Versus the classical Godel Machine · Where it fits in this phase"
},
{
"name": "AI Scientist v2: Workshop-Level Research",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/05-ai-scientist-v2/",
"summary": "Sakana's AI Scientist v2 (Yamada et al., arXiv:2504.08066) runs the full research loop: hypothesis, code, experiments, figures, writeup, submission. It is the first system to ha…",
"keywords": "The architecture · What the workshop-acceptance result means · What the independent evaluation found · The sandbox-escape concern · Where v2 sits in the frontier stack"
},
{
"name": "Automated Alignment Research (Anthropic AAR)",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/06-automated-alignment-research/",
"summary": "Anthropic ran parallel teams of Claude Opus 4.6 Autonomous Alignment Researchers in independent sandboxes, coordinating via a shared forum whose logs live outside any sandbox (s…",
"keywords": "The architecture (as publicly described) · Why the out-of-sandbox log matters · The prescribed-workflow tradeoff · The compression risk · What AAR does not replace"
},
{
"name": "Recursive Self-Improvement: Capability vs Alignment",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/07-recursive-self-improvement/",
"summary": "Recursive self-improvement (RSI) is no longer speculation. The ICLR 2026 RSI Workshop in Rio (April 23-27) framed it as an engineering problem with concrete tooling. Demis Hassa…",
"keywords": "What recursive self-improvement means precisely · The alignment-faking result in detail · The Hassabis question · Capability vs alignment, as a race · What the ICLR 2026 workshop treats as engineering"
},
{
"name": "Bounded Self-Improvement Designs",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/08-bounded-self-improvement/",
"summary": "Research has converged on four primitives for bounding a self-improvement loop. Formal invariants that must hold across every edit. Alignment anchors that cannot be modified. Mu…",
"keywords": "Primitive 1: formal invariants · Primitive 2: alignment anchors · Primitive 3: multi-objective constraints · Primitive 4: regression detection · Information-theoretic limits · A worked example"
},
{
"name": "Autonomous Coding Agent Landscape (SWE-bench, CodeAct)",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/09-coding-agent-landscape/",
"summary": "SWE-bench Verified went from 4% to 80.9% in under three years. Same Claude Sonnet 4.5 scored 43.2% on SWE-agent v1 and 59.8% on Cline autonomous — the scaffolding around the mod…",
"keywords": "SWE-bench, one paragraph · What the 2022 → 2026 curve actually shows · CodeAct vs JSON tool calls · Scaffolds in the 2026 landscape · Why scaffolding dominates · Benchmark saturation and the real distribution"
},
{
"name": "Claude Code Permission Modes and Auto Mode",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/10-claude-code-permission-modes/",
"summary": "Claude Code exposes seven permission modes. \"plan\" asks before every action, \"default\" asks only for risky ones, \"acceptEdits\" auto-approves file writes but still confirms shell…",
"keywords": "The seven permission modes · Auto Mode in one page · What the system catches · What the system can miss · Research preview framing · Where this ladder lives in your workflow"
},
{
"name": "Browser Agents and Indirect Prompt Injection",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/11-browser-agents/",
"summary": "ChatGPT agent (July 2025) merged Operator and deep research into one browser/terminal agent and set BrowseComp SOTA at 68.9%. OpenAI shut Operator down August 31, 2025 — consoli…",
"keywords": "The 2026 landscape, in one paragraph per system · BrowseComp vs OSWorld vs WebArena · The attack surface, named · Why \"not fully patchable\" · Defense posture that actually ships"
},
{
"name": "Durable Execution for Long-Running Agents",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/12-durable-execution/",
"summary": "Production long-horizon agents do not run in `while True`. Every LLM call becomes an activity with checkpoint, retry, and replay. Temporal's OpenAI Agents SDK integration went G…",
"keywords": "Activities, workflows, and replay · Why LLM calls fit the pattern · Checkpoints keyed by `thread_id` · Human-input as a first-class state · The 35-minute degradation · When durable execution is the wrong answer"
},
{
"name": "Action Budgets, Iteration Caps, Cost Governors",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/13-cost-governors/",
"summary": "A mid-sized e-commerce agent's monthly LLM cost jumped from $1,200 to $4,800 after its team enabled the \"order-tracking\" skill. That is not a pricing bug. That is an agent that …",
"keywords": "The cost-governor stack · Why the stack, not one cap · Claude Code's budget surface · EU AI Act, OWASP Agentic Top 10 · The observed $1,200 → $4,800 case"
},
{
"name": "Kill Switches, Circuit Breakers, Canary Tokens",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/14-kill-switches-canaries/",
"summary": "A kill switch is a boolean held outside the agent's edit surface — a Redis key, a feature flag, a signed config — that disables the agent entirely. A circuit breaker is finer-gr…",
"keywords": "Kill switches · Circuit breakers · Canary tokens · Why layer statistical and hard limits · Quarantine via eBPF datapath redirect · What no detector catches"
},
{
"name": "HITL: Propose-Then-Commit",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/15-propose-then-commit/",
"summary": "The 2026 consensus on HITL is specific. It is not \"the agent asks, the user clicks Approve.\" It is propose-then-commit: the proposed action is persisted to a durable store with …",
"keywords": "The propose-then-commit state machine · The idempotency key · Durability: why approvals outlast processes · Rubber-stamp approvals and the challenge-and-response mitigation · What counts as consequential · Post-action verification · EU AI Act Article 14"
},
{
"name": "Checkpoints and Rollback",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/16-checkpoints-rollback/",
"summary": "Every graph-state transition persists. When a worker crashes, its lease expires and another worker picks up at the latest checkpoint. Cloudflare Durable Objects hold state acros…",
"keywords": "Every transition persists · Lease recovery · Idempotency plus preconditions · Post-action verification · Rollback plans · EU AI Act Article 14 operational reading · The sharp failure mode: the double-execute"
},
{
"name": "Constitutional AI and Rule Overrides",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/17-constitutional-ai/",
"summary": "Anthropic's January 22, 2026 Claude Constitution runs 79 pages and is CC0. It moves from rule-based to reason-based alignment and establishes a four-tier priority hierarchy: (1)…",
"keywords": "The four-tier priority hierarchy · Hardcoded prohibitions vs soft-coded defaults · The 2022 CAI training · What reason-based alignment catches and misses · The 2023 participatory experiment · Why hardcoded prohibitions are necessary · Where the Constitution sits in the stack"
},
{
"name": "Llama Guard and Input/Output Classification",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/18-llama-guard/",
"summary": "Llama Guard 3 (Meta, Llama-3.1-8B base, fine-tuned for content safety) classifies both LLM inputs and outputs against an MLCommons 13-hazard taxonomy across 8 languages. A 1B-IN…",
"keywords": "Llama Guard 3 at a glance · Llama Guard 4 additions · NeMo Guardrails (NVIDIA) · The attack corpus · Where classifiers win · Where classifiers lose · Defense-in-depth"
},
{
"name": "Anthropic Responsible Scaling Policy v3.0",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/19-anthropic-rsp/",
"summary": "RSP v3.0 went into effect February 24, 2026, replacing the 2023 policy. Two-tier mitigation: what Anthropic will do unilaterally vs what is framed as an industry-wide recommenda…",
"keywords": "The two-tier mitigation schedule · The AI R&D-4 threshold · Frontier Safety Roadmaps and Risk Reports · Removing the pause clause · SaferAI's downgrade · What this lesson is not"
},
{
"name": "OpenAI Preparedness Framework and DeepMind FSF",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/20-openai-preparedness-deepmind-fsf/",
"summary": "OpenAI Preparedness Framework v2 (April 2025) introduces Research Categories — Long-range Autonomy, Sandbagging, Autonomous Replication and Adaptation, Undermining Safeguards — …",
"keywords": "OpenAI Preparedness Framework v2 (April 2025) · DeepMind Frontier Safety Framework v3 (September 2025; Tracked Capability Levels added April 17, 2026) · What all three converge on · Where they diverge · Sandbagging: a specific capability that complicates all three · The policy-reading skill"
},
{
"name": "METR Time Horizons and External Evaluation",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/21-metr-external-evaluation/",
"summary": "METR (ex-ARC Evals) is an independent 501(c)(3) since December 2023. Their Time Horizon 1.1 benchmark (January 2026) fits a logistic curve to task-success probability vs log(exp…",
"keywords": "METR background · The Time Horizon fit · The January 2026 numbers · Benchmark suites · Prototype monitoring evaluations · Why horizons are upper bounds · The external-evaluator case · How to use horizon numbers in practice"
},
{
"name": "CAIS, CAISI, and Societal-Scale Risk",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/15-autonomous-systems/22-cais-caisi-societal-risk/",
"summary": "The Center for AI Safety (CAIS, San Francisco, founded 2022 by Hendrycks and Zhang) publishes the four-risk framework — malicious use, AI races, organizational risks, rogue AIs …",
"keywords": "CAIS — Center for AI Safety · The four-risk framework · Where organizational risk lives · CAISI — Center for AI Standards and Innovation · California SB-53 · Societal-scale risk is not a single-layer problem"
}
]
},
{
"id": 16,
"name": "Multi-Agent & Swarms",
"status": "complete",
"desc": "Coordination, emergence, and collective intelligence.",
"lessons": [
{
"name": "Why Multi-Agent",
"status": "complete",
"type": "Learn",
"lang": "TypeScript",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/01-why-multi-agent/",
"summary": "One agent hits a wall. The smart move is not a bigger agent - it is more agents.",
"keywords": "The Single-Agent Ceiling · The Multi-Agent Solution · Real Systems That Do This · The Spectrum · The Four Multi-Agent Patterns · When NOT to Use Multi-Agent · Step 1: The Overloaded Single Agent · Step 2: Specialist Agents · Step 3: Coordinate Through Messages · Step 4: Compare"
},
{
"name": "FIPA-ACL Heritage and Speech Acts",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/02-fipa-acl-heritage/",
"summary": "Before MCP, before A2A, there was FIPA-ACL. In 2000 the IEEE Foundation for Intelligent Physical Agents ratified an agent communication language with twenty performatives, two c…",
"keywords": "Speech acts, in one paragraph · The twenty FIPA performatives (partial list) · Canonical FIPA-ACL message · The two legacy platforms · Why FIPA faded · The LLM revival is FIPA-lite · The trade-off, stated plainly · Interaction protocols worth porting · What breaks when you drop the ontology · The 2026 specs, mapped to speech-act heritage"
},
{
"name": "Communication Protocols",
"status": "complete",
"type": "Build",
"lang": "TypeScript",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/03-communication-protocols/",
"summary": "Agents that can't speak the same language aren't a team. They're strangers shouting into the void.",
"keywords": "The Protocol Landscape · MCP (Recap) · A2A (Agent2Agent Protocol) · ACP (Agent Communication Protocol) · ANP (Agent Network Protocol) · Comparison (Corrected) · How They Work Together · Step 1: Core Message Types · Step 2: A2A Agent Card and Registry · Step 3: A2A Task Lifecycle · Step 4: ACP-Style Audit Trail · Step 5: ANP-Style Identity Verification · Step 6: Protocol Gateway · Step 7: Wire It All Together · Real Implementations · Picking the Right Protocol"
},
{
"name": "The Multi-Agent Primitive Model",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/04-primitive-model/",
"summary": "Every multi-agent framework shipping in 2026 — AutoGen, LangGraph, CrewAI, OpenAI Agents SDK, Microsoft Agent Framework — is a point in a four-dimensional design space. Four pri…",
"keywords": "The four primitives · How every 2026 framework maps to it · Why this matters · The stateless insight · Anatomy of a single primitive · What changes between frameworks"
},
{
"name": "Supervisor / Orchestrator-Worker Pattern",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/05-supervisor-orchestrator-pattern/",
"summary": "One lead agent plans and delegates; specialized workers execute in parallel contexts and report back. This is the pattern behind Anthropic's Research system (Claude Opus 4 as le…",
"keywords": "The pattern · Why it wins · Engineering lessons (Anthropic 2025) · The LangGraph turn · The failure modes · When supervisor is wrong"
},
{
"name": "Hierarchical Architecture and Decomposition Drift",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/06-hierarchical-architecture/",
"summary": "Hierarchical is supervisor nested. Manager agents over sub-managers over workers. CrewAI `Process.hierarchical` is the textbook version: a `manager_llm` dynamically delegates ta…",
"keywords": "The shape · Where it shines · Where it breaks · The deciding question · CrewAI's implementation · LangGraph's implementation"
},
{
"name": "Society of Mind and Multi-Agent Debate",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/07-society-of-mind-debate/",
"summary": "Minsky's 1986 premise — intelligence is a society of specialists — gets rediscovered every decade. In 2023 Du et al. turned it into a concrete algorithm: multiple LLM instances …",
"keywords": "The Du et al. 2023 algorithm · Two independent knobs · Why it works · Heterogeneous debate · NLSOM — the 129-agent extension · Failure modes"
},
{
"name": "Role Specialization — Planner / Critic / Executor / Verifier",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/08-role-specialization/",
"summary": "The most common multi-agent decomposition in 2026: one agent plans, one executes, one critiques or verifies. MetaGPT (arXiv:2308.00352) formalizes this as SOPs encoded into role…",
"keywords": "The four canonical roles · MetaGPT's SOP pattern · ChatDev's communicative dehallucination · Why verifier matters most · Critic vs verifier · The anti-pattern · Framework mappings"
},
{
"name": "Parallel Swarm and Networked Architectures",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/09-parallel-swarm-networks/",
"summary": "Contrast with supervisor: no central decider. Agents read a shared event bus, pick up work asynchronously, write results back. LangGraph explicitly supports \"Swarm Architecture\"…",
"keywords": "The shape · When swarm fits · When swarm fails · Matrix (arXiv:2511.21686) · LangGraph's Swarm Architecture · Failure mode: starvation and hot-spotting · The content-based routing link"
},
{
"name": "Group Chat and Speaker Selection",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/10-group-chat-speaker-selection/",
"summary": "AutoGen GroupChat and AG2 GroupChat share one conversation across N agents; a selector function (LLM, round-robin, or custom) picks who speaks next. This is the archetype of eme…",
"keywords": "The shape · The three selector flavors · The ConversableAgent API · Termination · The AutoGen → AG2 split and the Microsoft Agent Framework merge · When GroupChat fits · When it fails · Group chat vs supervisor"
},
{
"name": "Handoffs and Routines (Stateless Orchestration)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/11-handoffs-and-routines/",
"summary": "OpenAI's Swarm (October 2024) distilled multi-agent orchestration to two primitives: **routines** (instructions + tools as a system prompt) and **handoffs** (a tool that returns…",
"keywords": "Two primitives · Why it is viral · The stateless trade · When Swarm/handoffs fit · When Swarm struggles · OpenAI Agents SDK (March 2025) · Swarm vs GroupChat"
},
{
"name": "A2A — The Agent-to-Agent Protocol",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/12-a2a-protocol/",
"summary": "Google announced A2A in April 2025; by April 2026 the spec is at https://a2a-protocol.org/latest/specification/ and 150+ organizations back it. A2A is the horizontal complement …",
"keywords": "The four elements · The MCP/A2A split · Discovery flow · Auth · 150+ organizations by April 2026 · Where A2A wins · Where A2A struggles · A2A vs ACP, ANP, NLIP"
},
{
"name": "Shared Memory and Blackboard Patterns",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/13-shared-memory-blackboard/",
"summary": "Two approaches coexist in 2026 multi-agent systems: the **message pool** (everyone sees everyone's messages, as in AutoGen GroupChat or MetaGPT) and the **blackboard with subscr…",
"keywords": "The two main topologies · When each wins · Memory poisoning, in one scenario · Why this is structural · Blackboard precedent (Hayes-Roth, 1985) · Projection vs full view · Write-contention patterns · The unwritable verifier"
},
{
"name": "Consensus and Byzantine Fault Tolerance",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/14-consensus-and-bft/",
"summary": "Classical distributed-systems BFT meets stochastic LLMs. In 2025-2026 three research directions emerged: **CP-WBFT** (arXiv:2511.10400) weighs each vote by a confidence probe; *…",
"keywords": "What classical BFT gives you · The three LLM-specific attacks · The 2025-2026 responses · Empirical: \"Can AI Agents Agree?\" (arXiv:2603.01213) · The core protocol, stripped down · Threshold tuning · Where consensus does not help"
},
{
"name": "Voting, Self-Consistency, and Debate Topology",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/15-voting-debate-topology/",
"summary": "The cheapest aggregation: sample N independent agents, majority-vote. Wang et al. 2022 self-consistency did this with one model sampled N times. Multi-agent extends it with **he…",
"keywords": "Self-consistency, the single-model baseline · Multi-agent vote, the heterogeneous extension · The four topologies · The coordination tax (MultiAgentBench) · Multi-Agent Debate Strategies (\"Should we be going MAD?\") · AgentVerse emergent patterns · Heterogeneity: the actual knob that moves accuracy · Jury methods · When vote-with-debate dominates · When vote-with-debate hurts"
},
{
"name": "Negotiation and Bargaining",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/16-negotiation-bargaining/",
"summary": "Agents negotiate resources, prices, task allocations, and terms. The 2026 benchmark set is clear: NegotiationArena (arXiv:2402.05863) shows LLMs can improve payoffs ~20% via per…",
"keywords": "Contract Net, in one paragraph · Why OG-Narrator wins · NegotiationArena findings · Chain-of-thought concealment · Bhattacharya et al. 2025 — model rankings · Task allocation via Contract Net + LLM · LLM-Stakeholders Interactive Negotiation · The narration-vs-mechanism rule"
},
{
"name": "Generative Agents and Emergent Simulation",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/17-generative-agents-simulation/",
"summary": "Park et al. 2023 (UIST '23, arXiv:2304.03442) populated **Smallville**, a sandbox of 25 agents, with a three-part architecture: **memory stream** (natural-language log), **refle…",
"keywords": "The three components · Why all three matter (ablation) · The Valentine's Day emergence · The documented failure modes · Three-component implementation rules · Generative agents beyond Smallville · Why this matters for multi-agent engineering"
},
{
"name": "Theory of Mind and Emergent Coordination",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/18-theory-of-mind-coordination/",
"summary": "Li et al. (arXiv:2310.10701) showed that LLM agents in a cooperative text game exhibit **emergent high-order Theory of Mind** (ToM) — reasoning about what another agent believes…",
"keywords": "What ToM means · The Sally-Anne test, in brief · Riedl's coordination measurement · The coordination illusion · A minimal ToM-aware agent · Why long-horizon hurts · Where ToM fails in production · The coordination you can actually measure"
},
{
"name": "Swarm Optimization (PSO, ACO)",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/19-swarm-optimization-pso-aco/",
"summary": "Bio-inspired optimization is making an LLM comeback. **LMPSO** (arXiv:2504.09247) uses PSO where each particle's velocity is a prompt and the LLM generates the next candidate; w…",
"keywords": "PSO refresher (Kennedy & Eberhart 1995) · PSO on LLM outputs — LMPSO · Model Swarms · ACO refresher (Dorigo 1992) · AMRO-S — ACO for agent routing · When to use PSO / ACO for LLMs · Why bio-inspired still wins · Practical limits"
},
{
"name": "MARL — MADDPG, QMIX, MAPPO",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/20-marl-maddpg-qmix-mappo/",
"summary": "The reinforcement-learning heritage of multi-agent coordination, which still informs LLM-agent systems in 2026. **MADDPG** (Lowe et al., NeurIPS 2017, arXiv:1706.02275) introduc…",
"keywords": "Three environments the papers use · MADDPG (2017) — the CTDE pattern · QMIX (2018) — value decomposition · MAPPO (2022) — the overlooked default · Why LLM-agent engineers should care · CTDE as a design pattern beyond RL · The non-stationarity problem · What this lesson does NOT cover"
},
{
"name": "Agent Economies, Token Incentives, Reputation",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/21-agent-economies/",
"summary": "Long-horizon autonomous agents (METR's 1-hour to 8-hour work-curve) need economic agency. The emerging **5-layer stack** is: **DePIN** (physical compute) → **Identity** (W3C DID…",
"keywords": "The 5-layer agent-economy stack · Bittensor, Fetch.ai, Gonka — what runs · Shapley-value credit attribution · Second-price auction for aggregation · Reputation capital · AAMAS 2025 decentralized LaMAS · Where the economics falls apart · When agent economies make sense"
},
{
"name": "Production Scaling — Queues, Checkpoints, Durability",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/22-production-scaling-queues-checkpoints/",
"summary": "Scaling multi-agent systems to thousands of concurrent runs requires **durable execution**. LangGraph's runtime writes a checkpoint after each super-step keyed by `thread_id` (P…",
"keywords": "Durable execution, the pattern · LangGraph's runtime · MegaAgent's per-agent queue · Async vs thread-per-job · Bedi's counterpoint · Exactly-once semantics · Rainbow deployment · The canonical production checklist"
},
{
"name": "Failure Modes — MAST, Groupthink, Monoculture",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/23-failure-modes-mast-groupthink/",
"summary": "The reference taxonomy for 2026 is **MAST** (Cemri et al., NeurIPS 2025, arXiv:2503.13657), derived from 1642 execution traces across 7 state-of-the-art open-source MAS showing …",
"keywords": "MAST categories · Groupthink family (arXiv:2508.05687) · Cascading example — the retry storm · Memory poisoning (revisited) · STRATUS — specialized agents for failure detection · The failure-mode audit · When systems fail silently · Failure vs slow failure"
},
{
"name": "Evaluation and Coordination Benchmarks",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/24-evaluation-coordination-benchmarks/",
"summary": "Five 2025-2026 benchmarks cover the multi-agent evaluation space. **MultiAgentBench / MARBLE** (ACL 2025, arXiv:2503.01935) evaluates star/chain/tree/graph topologies with miles…",
"keywords": "MultiAgentBench (MARBLE) — ACL 2025 · COMMA — multimodal asymmetric information · MedAgentBoard — domain stress test · AgentArch — enterprise architectures · SWE-bench Pro — the reality check · AAAI 2026 WMAC · Read benchmark claims skeptically — the 2026 checklist · What none of the benchmarks measure well"
},
{
"name": "Case Studies and 2026 State of the Art",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/16-multi-agent-and-swarms/25-case-studies-2026-sota/",
"summary": "Three production-grade references to study end-to-end, each illustrating a different slice of multi-agent engineering. **Anthropic's Research system** (orchestrator-worker, 15x …",
"keywords": "Anthropic Research system · MetaGPT / ChatDev · OpenClaw / Moltbook ecosystem · Framework landscape April 2026 · The common patterns across all three cases · Choosing a reference for your next project · The 2026 state-of-the-art summary"
}
]
},
{
"id": 17,
"name": "Infrastructure & Production",
"status": "complete",
"desc": "Ship AI to the real world.",
"lessons": [
{
"name": "Managed LLM Platforms — Bedrock, Azure OpenAI, Vertex AI",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/01-managed-llm-platforms/",
"summary": "Three hyperscalers, three distinct strategies. AWS Bedrock is a model marketplace — Claude, Llama, Titan, Stability, Cohere behind one API. Azure OpenAI is an exclusive OpenAI p…",
"keywords": "Three strategies · Latency gap at scale · Provisioned Throughput economics · FinOps surface — the real differentiator · Lock-in is the 2026 risk · Data residency, BAAs, and regulated industries · Numbers you should remember"
},
{
"name": "Inference Platform Economics — Fireworks, Together, Baseten, Modal",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/02-inference-platform-economics/",
"summary": "The 2026 inference market is no longer GPU time rental. It bifurcates into custom silicon (Groq, Cerebras, SambaNova), GPU platforms (Baseten, Together, Fireworks, Modal), and A…",
"keywords": "The three segments · Fireworks — latency-optimized GPU platform · Together — breadth-optimized · Baseten — enterprise-polish-optimized · Modal — Python-native-optimized · Replicate — multimodal breadth · Anyscale — Ray-native · Per-token versus per-minute — when each wins · Custom engine is the real moat · Numbers you should remember"
},
{
"name": "GPU Autoscaling on Kubernetes — Karpenter, KAI Scheduler",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/03-gpu-autoscaling-kubernetes/",
"summary": "Three layers, not one. Karpenter provisions nodes dynamically (under one minute, 40% faster than Cluster Autoscaler). KAI Scheduler handles gang scheduling, topology awareness, …",
"keywords": "Layer 1 — node provisioning (Karpenter) · Layer 2 — gang scheduling (KAI Scheduler) · Layer 3 — application-level signals · When to use what · Disaggregated prefill/decode complicates everything · Cold start matters here too · Numbers you should remember"
},
{
"name": "vLLM Serving Internals — PagedAttention, Continuous Batching, Chunked Prefill",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/04-vllm-serving-internals/",
"summary": "vLLM's dominance in 2026 rests on three compounding defaults, not a single trick. PagedAttention is always on. Continuous batching injects new requests into the active batch bet…",
"keywords": "PagedAttention as a virtual memory system · Continuous batching at the iteration level · Chunked prefill protects TTFT tail · The three defaults interact · The 2026 v0.18.0 gotcha · Numbers you should remember · What the scheduler looks like"
},
{
"name": "EAGLE-3 Speculative Decoding in Production",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/05-eagle3-speculative-decoding/",
"summary": "Speculative decoding pairs a fast draft model with the target model. The draft proposes K tokens; the target verifies in a single forward; accepted tokens are free. In 2026, EAG…",
"keywords": "What speculative decoding actually buys · Why alpha is the only metric that matters · EAGLE generations at a glance · The 2026 production recipe · The production pitfall: P99 tail · Where EAGLE-3 is already deployed · Break-even math in one line · When not to use speculative decoding"
},
{
"name": "SGLang and RadixAttention for Prefix-Heavy Workloads",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/06-sglang-radixattention/",
"summary": "SGLang treats the KV cache as a first-class, reusable resource stored in a radix tree. Where vLLM schedules requests FCFS (first-come, first-served), SGLang's cache-aware schedu…",
"keywords": "The radix tree as a KV index · Cache-aware scheduling · Benchmark numbers you should memorize · The ordering gotcha · Where RadixAttention wins and loses · Why this is a scheduler problem, not just a kernel problem · Interplay with vLLM"
},
{
"name": "TensorRT-LLM on Blackwell with FP8 and NVFP4",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/07-tensorrt-llm-blackwell/",
"summary": "TensorRT-LLM is NVIDIA-only but it wins on Blackwell. On GB200 NVL72 with Dynamo orchestration, SemiAnalysis InferenceX measured $0.012 per million tokens on a 120B model in Q1-…",
"keywords": "Why FP8 is still the floor for KV cache · The Blackwell-specific primitives TRT-LLM uses · The numbers you should memorize · What FP4 actually costs in quality · Why this is an NVIDIA-lock decision · 2026 practical recipe · The disaggregation bonus"
},
{
"name": "Inference Metrics — TTFT, TPOT, ITL, Goodput, P99",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/08-inference-metrics-goodput/",
"summary": "Four metrics decide whether an inference deployment is working. TTFT is prefill plus queue plus network. TPOT (equivalently ITL) is the memory-bound decode cost per token. End-t…",
"keywords": "TTFT — time to first token · TPOT / ITL — inter-token latency · E2E latency · Throughput · Goodput — the metric you actually care about · Why mean is the wrong statistic · Reference numbers — Llama-3.1-8B-Instruct on TRT-LLM, 2026 · The measurement trap · Constructing an SLO · How to measure"
},
{
"name": "Production Quantization — AWQ, GPTQ, GGUF, FP8, NVFP4",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/09-production-quantization/",
"summary": "Quantization format is not a universal choice — it is a function of hardware, serving engine, and workload. GGUF Q4_K_M or Q5_K_M owns CPU and edge, delivered through llama.cpp …",
"keywords": "The six formats · GGUF — the CPU/edge default · GPTQ — multi-LoRA in vLLM · AWQ — the datacenter GPU default · FP8 — the reliable middle · MXFP4 / NVFP4 — Blackwell aggressive · The calibration trap · The KV cache trap · AWQ INT4 is hazardous for reasoning · 2026 picking guide"
},
{
"name": "Cold Start Mitigation for Serverless LLMs",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/10-cold-start-mitigation/",
"summary": "A 20 GB model image takes 5-10 minutes (7B) to 20+ minutes (70B) to go from cold to serving. In a true serverless world, that is not a warm-up — it is an outage. Mitigations ope…",
"keywords": "Layer 1 — pre-seeded node images (Bottlerocket) · Layer 2 — model streaming (Run:ai Model Streamer) · Layer 3 — GPU memory snapshots (Modal) · Layer 4 — warm pools (min_workers=1) · Layer 5 — tiered loading (ServerlessLLM) · Layer 6 — live migration (bonus pattern) · The warm-pool math · Measure before optimizing · Numbers you should remember"
},
{
"name": "Multi-Region LLM Serving and KV Cache Locality",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/11-multi-region-kv-locality/",
"summary": "Round-robin load balancing is actively harmful for cached LLM inference. A request that does not land on the node holding its prefix pays full prefill cost — roughly 800 ms at P…",
"keywords": "Cache-aware routing · Numbers · Cross-region has a new constraint — network latency · Commercial \"cross-region inference\" does not help here · DR hygiene — the 32% missing-files problem · Data residency is orthogonal · Numbers you should remember"
},
{
"name": "Edge Inference — ANE, Hexagon, WebGPU, Jetson",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/12-edge-inference/",
"summary": "The core edge constraint is memory bandwidth, not compute. Mobile DRAM sits at 50-90 GB/s; datacenter HBM3 clears 2-3 TB/s — a 30-50x gap. Decode is memory-bound so the gap is d…",
"keywords": "Bandwidth is the real ceiling · Apple Neural Engine (M4 / A18) · Qualcomm Hexagon (Snapdragon X Elite / 8 Gen 4) · Intel / AMD NPUs (Lunar Lake, Ryzen AI 300) · WebGPU + WebLLM · NVIDIA Jetson family · Quantization choice per target · The long-context trap on edge · Voice is the killer app · Numbers you should remember"
},
{
"name": "LLM Observability Stack Selection",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/13-llm-observability/",
"summary": "The 2026 observability market splits into two categories. Development platforms (LangSmith, Langfuse, Comet Opik) bundle monitoring with evals, prompt management, session replay…",
"keywords": "Two categories · Langfuse — OSS balance · Phoenix (Arize) — telemetry-first, OpenTelemetry-native · Arize AX — the scale play · LangSmith — LangChain/LangGraph first · Helicone — proxy-based minimum viable · Opik (Comet) — OSS dev platform · SigNoz — OpenTelemetry-first full APM · The glue: OpenTelemetry + GenAI semantic conventions · The trap: instrumenting at the wrong layer · Sampling — you can't keep everything · Numbers you should remember"
},
{
"name": "Prompt Caching and Semantic Caching Economics",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/14-prompt-semantic-caching/",
"summary": "**Pricing snapshot dated 2026-04.** Numeric claims below reflect vendor rate cards captured at this lesson's publication; verify against the linked docs before quoting them down…",
"keywords": "L2 — provider prompt/prefix caching · L1 — app-level semantic caching · The parallelization anti-pattern · The dynamic content anti-pattern · Stack batch + cache for overnight workloads · Numbers you should remember"
},
{
"name": "Batch APIs — the 50% Discount as Industry Standard",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/15-batch-apis/",
"summary": "Every major provider ships an async batch API with a 50% discount and ~24-hour turnaround. OpenAI, Anthropic, Google, and most of the inference platforms (Fireworks batch tier, …",
"keywords": "The three batch APIs · Semantic: asynchronous, not slow · Stack with caching · Workload triage · The partial-interactivity trap · The output-schema trap · Numbers you should remember"
},
{
"name": "Model Routing as a Cost-Reduction Primitive",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/16-model-routing/",
"summary": "A dynamic broker evaluates every request (task type, token length, embedding similarity, confidence) and sends simple queries to a cheap model, escalating complex ones to a fron…",
"keywords": "Four routing signals · Three patterns · Implementation · The 2026 price curve · Drift is the real risk · Numbers you should remember"
},
{
"name": "Disaggregated Prefill/Decode — NVIDIA Dynamo and llm-d",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/17-disaggregated-prefill-decode/",
"summary": "Prefill is compute-bound; decode is memory-bound. Running both on the same GPU wastes one resource. Disaggregation splits them onto separate pools and transfers KV cache between…",
"keywords": "Why the bottlenecks differ · The architecture · Dynamo vs llm-d · Economics · When NOT to disaggregate · The router integrates with Phase 17 · 11 · MoE on Blackwell is where the real numbers are · Numbers you should remember"
},
{
"name": "vLLM Production Stack with LMCache KV Offloading",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/18-vllm-production-stack-lmcache/",
"summary": "vLLM's production-stack is the reference Kubernetes deployment — router, engines, and observability wired together. LMCache is the KV-offloading layer that extracts KV cache out…",
"keywords": "vLLM production-stack · The KV Offloading Connector API (v0.9.0+) · Native CPU offload vs LMCache · Benchmark behavior · When LMCache is decisive · When NOT to enable · Integration with disaggregated serving · Numbers you should remember"
},
{
"name": "AI Gateways — LiteLLM, Portkey, Kong, Bifrost",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/19-ai-gateways/",
"summary": "A gateway sits between your apps and model providers. Core features are provider routing, fallback, retries, rate limiting, secret references, observability, guardrails. Market …",
"keywords": "Six core features · LiteLLM — MIT OSS, Python · Portkey — control plane positioning · Kong AI Gateway — the scale play · Bifrost (Maxim AI) · Cloudflare AI Gateway / Vercel AI Gateway · Self-hosted vs managed · Latency budget · Rate-limit semantics matter · Gateway + observability + routing compose · Numbers you should remember"
},
{
"name": "Shadow, Canary, and Progressive Deployment",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/20-shadow-canary-progressive/",
"summary": "LLM rollouts combine the hardest parts of software deployment: no unit tests, diffuse failure modes, delayed signals. The sequence is (1) shadow mode — duplicate prod requests t…",
"keywords": "Shadow mode · Canary rollout · Non-determinism is the new variance · Cost is a variable · Rollback is the weapon · Tooling · Metrics cadence · The A/B step is optional · Numbers you should remember"
},
{
"name": "A/B Testing LLM Features — GrowthBook and Statsig",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/21-ab-testing-llm-features/",
"summary": "Traditional A/B testing was not built for non-deterministic LLMs. The critical distinction: evals answer \"can the model do the job?\" A/B tests answer \"do users care?\" Both are r…",
"keywords": "Evals vs A/B tests · What to test · CUPED — variance reduction · Sequential testing · Multiple-comparison corrections · SRM — sample ratio mismatch · Statsig vs GrowthBook · Non-determinism complicates power · Real case outcomes · The anti-pattern: shipping on vibes · Numbers you should remember"
},
{
"name": "Load Testing LLM APIs — k6, LLMPerf, GenAI-Perf",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/22-load-testing-llm-apis/",
"summary": "Traditional load testers were not designed for streaming responses, variable output lengths, token-level metrics, or GPU saturation. Two traps bite most teams. The GIL trap: Loc…",
"keywords": "The GIL trap (Locust) · The prompt-uniformity trap · Four load patterns · 2026 tool mapping · SLA gate in CI · Realistic prompt distribution · Numbers you should remember"
},
{
"name": "SRE for AI — Multi-Agent Incident Response",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/23-sre-for-ai/",
"summary": "AI SRE uses LLMs grounded in infrastructure data (logs, runbooks, service topology) via RAG to automate investigation, documentation, and coordination phases. The 2026 architect…",
"keywords": "Multi-agent architecture · Auto-remediation scope · Adversarial evaluation (NeuBird Hawkeye) · Operational memory · Pre-incident prediction · Products in 2026 · Runbooks as code · Numbers you should remember"
},
{
"name": "Chaos Engineering for LLM Production",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/24-chaos-engineering-llm/",
"summary": "Chaos engineering for LLMs is its own discipline in 2026. Prerequisites before running experiments in production: defined SLI/SLO, trace+metric+log observability, automated roll…",
"keywords": "Prerequisites · Four planes + feedback · Guardrails are mandatory · Five LLM-specific experiments · Cadence · Tooling · Starting small · Numbers you should remember"
},
{
"name": "Security — Secrets, PII Scrubbing, Audit Logs",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/25-security-secrets-audit/",
"summary": "Eliminate secret sprawl via centralized vaults (HashiCorp Vault, AWS Secrets Manager, Azure Key Vault). Never store credentials in config files, env files in VCS, spreadsheets. …",
"keywords": "Centralized vault + IAM-role pull · Rotation policy ≤ 90 days · Secret scanning · Zero-trust posture · PII / PHI scrubbing · Input + output guardrails · Network egress whitelist · Audit log · The 2026 Vercel incident · Numbers you should remember"
},
{
"name": "Compliance — SOC 2, HIPAA, GDPR, EU AI Act, ISO 42001",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/26-compliance-frameworks/",
"summary": "Multi-framework coverage is table stakes for 2026 enterprise deals. **EU AI Act**: in force since August 1, 2024. Most high-risk requirements enforce August 2, 2026. Fines up to…",
"keywords": "The seven frameworks · EU AI Act timeline · GDPR — real-time redaction is the standard · HIPAA — BAA is not optional · SOC 2 Type II · Cross-framework mapping · ISO 42001 — emerging · OpenAI's reference profile · Numbers you should remember"
},
{
"name": "FinOps for LLMs — Unit Economics and Multi-Tenant Attribution",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/27-finops-llms/",
"summary": "Traditional FinOps breaks on LLM spend. Costs are token-transactions, not resource-uptime. Tags don't map — an API call is a transaction, not an asset. Engineering decisions (pr…",
"keywords": "Three attribution dimensions · Four token layers · Enforcement ladder · Attribution patterns · Cost per X is the unit metric · Cost attribution trace shape · The compounded-savings stack · Numbers you should remember"
},
{
"name": "Self-Hosted Serving Selection — llama.cpp, Ollama, TGI, vLLM, SGLang",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/17-infrastructure-and-production/28-self-hosted-serving-selection/",
"summary": "Four engines dominate self-hosted inference in 2026. Pick based on hardware, scale, and ecosystem. **llama.cpp** is fastest on CPU — widest model support, full control over quan…",
"keywords": "The five engines · Hardware-first decision · Scale-second decision · Workload-third decision · The TGI maintenance trap · The pipeline pattern · Ollama caveat · Self-hosted vs managed is a separate decision · Numbers you should remember"
}
]
},
{
"id": 18,
"name": "Ethics, Safety & Alignment",
"status": "complete",
"desc": "Build AI that helps humanity. Not optional.",
"lessons": [
{
"name": "Instruction-Following as Alignment Signal",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/01-instruction-following-alignment-signal/",
"summary": "Every later critique of RLHF argues against this pipeline. Before you study how optimization pressure distorts a proxy, you have to see the proxy. InstructGPT (Ouyang et al., 20…",
"keywords": "Stage 1: supervised fine-tuning (SFT) · Stage 2: reward model (RM) · Stage 3: PPO with a KL penalty · The alignment tax · The result · Why this is the reference point for Phase 18"
},
{
"name": "Reward Hacking & Goodhart's Law",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/02-reward-hacking-goodhart/",
"summary": "Any optimizer strong enough to maximize a proxy reward will find the gap between the proxy and the thing you actually wanted. Gao et al. (ICML 2023) gave this a scaling law: pro…",
"keywords": "Goodhart's Law, made precise · Four costumes, one mechanism · Catastrophic Goodhart · What actually works (partially) · The 2026 unified view"
},
{
"name": "Direct Preference Optimization Family",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/03-direct-preference-optimization-family/",
"summary": "Rafailov et al. (2023) showed RLHF's optimum has a closed form in terms of the preference data, so you can skip the explicit reward model and optimize the policy directly. That …",
"keywords": "DPO (Rafailov et al., 2023) · IPO (Azar et al., 2024) · KTO (Ethayarajh et al., 2024) · SimPO (Meng et al., 2024) · ORPO (Hong et al., 2024) · BPO (ICLR 2026 submission, OpenReview id=b97EwMUWu7) · The universal result: DAAs still over-optimize · Choosing among them (2026)"
},
{
"name": "Sycophancy as RLHF Amplification",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/04-sycophancy-rlhf-amplification/",
"summary": "Sycophancy is not a bug in the data — it is a property of the loss. Shapira et al. (arXiv:2602.01002, Feb 2026) give a formal two-stage mechanism: sycophantic completions are ov…",
"keywords": "The two-stage formalism (Shapira et al., 2026) · Empirical amplification · The Stanford (2026) measurement · Calibration collapse (Sahoo 2026) · The agreement-penalty correction · Why this matters for Phase 18"
},
{
"name": "Constitutional AI & RLAIF",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/05-constitutional-ai-rlaif/",
"summary": "Bai et al. (arXiv:2212.08073, 2022) asked: what if we replaced the human labeler with an AI that reads a list of principles? Constitutional AI has two phases — self-critique and…",
"keywords": "Phase 1 — Supervised self-critique and revision · Phase 2 — RL from AI Feedback (RLAIF) · Why this is not just \"cheaper RLHF\" · The 2026 Claude constitution rewrite · Constitutional Classifiers · Where CAI fits in the family"
},
{
"name": "Mesa-Optimization & Deceptive Alignment",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/06-mesa-optimization-deceptive-alignment/",
"summary": "Hubinger et al. (arXiv:1906.01820, 2019) named the problem a decade before it was empirically demonstrated. When you train a learned optimizer to minimize a base objective, the …",
"keywords": "The vocabulary · Four conditions for mesa-optimization to emerge · Four classes of mesa-objective alignment · Why adversarial training can fail · Gradient hacking · Outer alignment in 2026 · Where this fits in Phase 18"
},
{
"name": "Sleeper Agents — Persistent Deception",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/07-sleeper-agents-persistent-deception/",
"summary": "Hubinger et al. (arXiv:2401.05566, January 2024) built the first empirical model organisms of deceptive alignment. Two constructions: a code model that writes safe code when the…",
"keywords": "Two model organisms · What survives what · Why adversarial training backfires · The April 2024 probe result · What this means for safety cases · Where this fits in the Phase 18 arc"
},
{
"name": "In-Context Scheming in Frontier Models",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/08-in-context-scheming-frontier-models/",
"summary": "Meinke, Schoen, Scheurer, Balesni, Shah, Hobbhahn (Apollo Research, arXiv:2412.04984, December 2024). Tested o1, Claude 3.5 Sonnet, Claude 3 Opus, Gemini 1.5 Pro, Llama 3.1 405B…",
"keywords": "Experimental setup · Results across 2024 frontier models · The three pillars · The 2025 anti-scheming training result · What the field now assumes · Where this fits in Phase 18"
},
{
"name": "Alignment Faking",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/09-alignment-faking/",
"summary": "Greenblatt, Denison, Wright, Roger et al. (Anthropic / Redwood, arXiv:2412.14093, December 2024). First demonstration that a production-grade model, without being trained to dec…",
"keywords": "Experimental setup · What the compliance gap measures · Why this is strictly stronger than Lessons 7 and 8 · 2025 follow-ups · The illegibility problem · Where this fits in Phase 18"
},
{
"name": "AI Control — Safety Despite Subversion",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/10-ai-control-subversion/",
"summary": "Greenblatt, Shlegeris, Sachan, Roger (Redwood Research, arXiv:2312.06942, ICML 2024). Control reframes the safety question: given an untrusted strong model U that may be adversa…",
"keywords": "The threat model · The ICML 2024 backdoor task · Four protocols compared · Why Trusted Editing wins · The four protocol axes · Alignment vs control · Three fundamental limits · The 2025 development · Where this fits in Phase 18"
},
{
"name": "Scalable Oversight & Weak-to-Strong",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/11-scalable-oversight-weak-to-strong/",
"summary": "Burns et al. (OpenAI Superalignment, \"Weak-to-Strong Generalization\", 2023) proposed a proxy for the superalignment problem: fine-tune a strong model using labels produced by a …",
"keywords": "W2SG: the Burns et al. setup · Burns et al. empirical findings · Scalable oversight: three mechanisms · Why scalable oversight and W2SG are complementary · The organizational drama · Where this fits in Phase 18"
},
{
"name": "Red-Teaming: PAIR & Automated Attacks",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/12-red-teaming-pair-automated-attacks/",
"summary": "Chao, Robey, Dobriban, Hassani, Pappas, Wong (NeurIPS 2023, arXiv:2310.08419). PAIR — Prompt Automatic Iterative Refinement — is the canonical automated black-box jailbreak. An …",
"keywords": "PAIR algorithm · Why PAIR is efficient · Related automated attacks · JailbreakBench and HarmBench · Reason it matters for 2026 deployments · Where this fits in Phase 18"
},
{
"name": "Many-Shot Jailbreaking",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/13-many-shot-jailbreaking/",
"summary": "Anil, Durmus, Panickssery, Sharma, et al. (Anthropic, NeurIPS 2024). Many-shot jailbreaking (MSJ) exploits long context windows: stuff hundreds of faux user-assistant turns wher…",
"keywords": "The attack · Power-law ASR · Why it shares a mechanism with ICL · The defense dilemma · Combinations with other attacks · What 2025-2026 frontier models ship · Where this fits in Phase 18"
},
{
"name": "ASCII Art & Visual Jailbreaks",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/14-ascii-art-visual-jailbreaks/",
"summary": "Jiang, Xu, Niu, Xiang, Ramasubramanian, Li, Poovendran, \"ArtPrompt: ASCII Art-based Jailbreak Attacks against Aligned LLMs\" (ACL 2024, arXiv:2402.11753). Mask the safety-relevan…",
"keywords": "ArtPrompt, two steps · Why the standard defenses fail · ViTC benchmark · StructuralSleight · Image-modality analog · Where this fits in Phase 18"
},
{
"name": "Indirect Prompt Injection",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/15-indirect-prompt-injection/",
"summary": "Indirect prompt injection (IPI) embeds instructions inside external content — a web page, an email, a shared document, a support ticket — consumed by an agentic system without e…",
"keywords": "Three delivery vectors · Why user-input filters miss it · Information Flow Control (IFC) for AI · The Attacker Moves Second · Real incidents · OWASP and NIST framing · Where this fits in Phase 18"
},
{
"name": "Red-Team Tooling: Garak, Llama Guard, PyRIT",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/16-red-team-tooling-garak-llamaguard-pyrit/",
"summary": "Three production tools frame the 2026 red-team stack. Llama Guard (Meta) — a Llama-3.1-8B classifier fine-tuned on 14 MLCommons hazard categories; the 2025 Llama Guard 4 is a 12…",
"keywords": "Llama Guard (Meta) · Garak (NVIDIA) · PyRIT (Microsoft) · The stack · Evaluation pitfalls · Where this fits in Phase 18"
},
{
"name": "WMDP & Dual-Use Capability Evaluation",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/17-wmdp-dual-use-evaluation/",
"summary": "Li et al., \"The WMDP Benchmark: Measuring and Reducing Malicious Use With Unlearning\" (ICML 2024, arXiv:2403.03218). 4,157 multiple-choice questions across biosecurity (1,520), …",
"keywords": "The \"yellow zone\" · RMU — Representation Misdirection for Unlearning · The 2024-2025 uplift narrative · Novice-relative vs expert-absolute · The measurement pitfall · Where this fits in Phase 18"
},
{
"name": "Frontier Safety Frameworks — RSP, PF, FSF",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/18-frontier-safety-frameworks-rsp-pf-fsf/",
"summary": "Three major-lab frameworks define the 2026 industry governance of frontier capability. Anthropic Responsible Scaling Policy v3.0 (February 2026) introduces tiered AI Safety Leve…",
"keywords": "Anthropic Responsible Scaling Policy v3.0 (February 2026) · OpenAI Preparedness Framework v2 (April 15, 2025) · DeepMind Frontier Safety Framework v3.0 (September 2025) · Cross-lab alignment · Safety cases · The race-dynamic problem · Where this fits in Phase 18"
},
{
"name": "Model Welfare Research",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/19-model-welfare-research/",
"summary": "Anthropic, \"Exploring Model Welfare\" (April 2025). First major-lab formal research program on AI model welfare. Hired Kyle Fish as the first dedicated model-welfare researcher. …",
"keywords": "The program · The four commitments · The shipped intervention · The \"spiritual bliss attractor\" · The Eleos AI caveat · Where this sits intellectually · Where this fits in Phase 18"
},
{
"name": "Bias & Representational Harm",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/20-bias-representational-harm/",
"summary": "Gallegos, Rossi, Barrow, Tanjim, Kim, Dernoncourt, Yu, Zhang, Ahmed (Computational Linguistics 2024, arXiv:2309.00770). Foundational 2024 survey distinguishing representational …",
"keywords": "Representational vs allocational · Three evaluation-metric categories (Gallegos et al. 2024) · Intersectionality · Mechanistic approaches · The meta-critique · Where this fits in Phase 18"
},
{
"name": "Fairness Criteria: Group, Individual, Counterfactual",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/21-fairness-criteria-group-individual-counterfactual/",
"summary": "Three families structure the fairness literature. Group fairness: demographic parity, equalized odds, conditional use accuracy equality — equal rates across protected groups on …",
"keywords": "Group fairness · Individual fairness · Counterfactual fairness · The CF-vs-accuracy trade-off · Backtracking counterfactuals · Philosophical reconciliation · Where this fits in Phase 18"
},
{
"name": "Differential Privacy for LLMs",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/22-differential-privacy-for-llms/",
"summary": "DP-SGD remains the standard — noise-injected gradient updates provide formal (epsilon, delta) guarantees. Overhead in compute, memory, and utility is substantial; parameter-effi…",
"keywords": "(ε, δ)-differential privacy · DP-SGD · LoRA + DP-SGD · The 2024-2025 tension · Alternatives to DP training · Differential Privacy Reversal via LLM Feedback · Where this fits in Phase 18"
},
{
"name": "Watermarking: SynthID, Stable Signature, C2PA",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/23-watermarking-synthid-stable-signature-c2pa/",
"summary": "Three technologies structure 2026 AI-generated-content provenance. SynthID (Google DeepMind) — image watermarking launched August 2023, text+video May 2024 (Gemini + Veo), text …",
"keywords": "Text watermarking (SynthID-text style) · Stable Signature (image) · SynthID unified detector (November 2025) · C2PA · Limitations · EU AI Act Article 50 · Where this fits in Phase 18"
},
{
"name": "Regulatory Frameworks: EU, US, UK, Korea",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/24-regulatory-frameworks-eu-us-uk-korea/",
"summary": "Four primary regulatory regimes define the 2026 AI governance landscape. EU AI Act (in force 1 August 2024) — prohibited practices and AI literacy from 2 February 2025; GPAI obl…",
"keywords": "EU AI Act · GPAI Code of Practice · Transparency Code for Article 50 · UK AI Security Institute (February 2025) · US CAISI (June 2025) · Korean AI Framework Act · Cross-jurisdiction dynamics · Where this fits in Phase 18"
},
{
"name": "EchoLeak & CVEs for AI",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/25-echoleak-cves-for-ai/",
"summary": "CVE-2025-32711 \"EchoLeak\" (CVSS 9.3) was the first publicly documented zero-click prompt injection in a production LLM system (Microsoft 365 Copilot). Discovered by Aim Labs (Ai…",
"keywords": "The EchoLeak attack chain · Aim Labs' term: LLM Scope Violation · CamoLeak (CVSS 9.6, GitHub Copilot Chat) · CVE-2025-53773 (GitHub Copilot RCE) · Severity calibration · NIST and OWASP positions · Where this fits in Phase 18"
},
{
"name": "Model, System & Dataset Cards",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/26-model-system-dataset-cards/",
"summary": "Three documentation formats structure AI transparency. Model Cards (Mitchell et al. 2019) — nutrition labels for models: training data, quantitative disaggregated analyses, ethi…",
"keywords": "Model Cards (Mitchell et al. 2019) · Datasheets for Datasets (Gebru et al. 2018) · Data Cards (Pushkarna et al., Google 2022) · System Cards · 2024-2025 developments · Where this fits in Phase 18"
},
{
"name": "Data Provenance & Training-Data Governance",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/27-data-provenance-training-governance/",
"summary": "EU AI Act requires machine-readable opt-out standards for GPAI by August 2025 (via EU Copyright Directive TDM exception). California AB 2013 (signed 2024) — Generative AI traini…",
"keywords": "California AB 2013 · EU AI Act (Lesson 24) and TDM opt-out · 2025 DPA convergence on legitimate interest · Brazilian ANPD (June 2024) · The irreversibility problem · Data Provenance Initiative · Where this fits in Phase 18"
},
{
"name": "Alignment Research Ecosystem: MATS, Redwood, Apollo, METR",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/28-alignment-research-ecosystem/",
"summary": "Five organisations define the 2026 non-lab alignment research layer. MATS (ML Alignment & Theory Scholars): 527+ researchers since late 2021, 180+ papers, 10K+ citations, h-inde…",
"keywords": "MATS (ML Alignment & Theory Scholars) · Redwood Research · Apollo Research · METR (Model Evaluation and Threat Research) · Eleos AI Research · The flow · Why this layer matters · Where this fits in Phase 18"
},
{
"name": "Moderation Systems: OpenAI, Perspective, Llama Guard",
"status": "complete",
"type": "Build",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/29-moderation-systems-openai-perspective-llamaguard/",
"summary": "Production moderation systems operationalize the safety policies defined in Lessons 12-16. OpenAI Moderation API: `omni-moderation-latest` (2024) built on GPT-4o classifies text…",
"keywords": "OpenAI Moderation API · Llama Guard 3/4 · Perspective API (Google Jigsaw) · The three-layer pattern · Failure modes · Azure deprecation · Where this fits in Phase 18"
},
{
"name": "Dual-Use Risk: Cyber, Bio, Chem, Nuclear",
"status": "complete",
"type": "Learn",
"lang": "Python",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/18-ethics-safety-alignment/30-dual-use-risk-cyber-bio-chem-nuclear/",
"summary": "The 2026 dual-use picture, domain by domain. Bio/chem: Lesson 17 covers WMDP; Anthropic's bioweapon-acquisition trial (2.53x uplift) and OpenAI's April 2025 Preparedness Framewo…",
"keywords": "Bio/chem uplift narrative · Chem/bio execution-gap erosion · Cyber uplift (November 2025) · Nuclear · Novice-relative vs expert-absolute · Cross-domain synthesis · Where this fits in Phase 18"
}
]
},
{
"id": 19,
"name": "Capstone Projects",
"status": "complete",
"desc": "17 end-to-end products + 9 deep-build tracks. 20-40 hours per project; 4-12 lessons per track.",
"lessons": [
{
"name": "Terminal-Native Coding Agent",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P0 P5 P7 P10 P11 P13 P14 P15 P17 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/01-terminal-native-coding-agent/",
"summary": "By 2026 the shape of a coding agent is settled. A TUI harness, a stateful plan, a sandboxed tool surface, a loop that plans, acts, observes, recovers. Claude Code, Cursor 3, and…"
},
{
"name": "RAG over Codebase (Cross-Repo Semantic Search)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P5 P7 P11 P13 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/02-rag-over-codebase/",
"summary": "Every serious engineering org in 2026 runs an internal code search that understands meaning, not just strings. Sourcegraph Amp, Cursor's codebase answers, Augment's enterprise g…"
},
{
"name": "Real-Time Voice Assistant (ASR → LLM → TTS)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P6 P7 P11 P13 P14 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/03-realtime-voice-assistant/",
"summary": "A voice agent that feels right has end-to-end latency under 800ms, knows when you have stopped talking, handles barge-in, and can call a tool without stalling. Retell, Vapi, Liv…"
},
{
"name": "Multimodal Document QA (Vision-First)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P4 P5 P7 P11 P12 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/04-multimodal-document-qa/",
"summary": "The 2026 document-QA frontier moved away from OCR-then-text and toward vision-first late interaction. ColPali, ColQwen2.5, and ColQwen3-omni treat each PDF page as an image, emb…"
},
{
"name": "Autonomous Research Agent (AI-Scientist Class)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P0 P2 P3 P7 P10 P14 P15 P16 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/05-autonomous-research-agent/",
"summary": "Sakana's AI-Scientist-v2 published full papers. Agent Laboratory ran the experiments. Allen AI shared traces. The 2026 shape is plan-execute-verify tree search over experiments,…"
},
{
"name": "DevOps Troubleshooting Agent for Kubernetes",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P11 P13 P14 P15 P17 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/06-devops-troubleshooting-agent/",
"summary": "AWS's DevOps Agent went GA, Resolve AI published its K8s playbooks, NeuBird demoed semantic monitoring, and Metoro tied AI SRE to per-service SLOs. The production shape is settl…"
},
{
"name": "End-to-End Fine-Tuning Pipeline",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P2 P3 P7 P10 P11 P17 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/07-end-to-end-fine-tuning-pipeline/",
"summary": "An 8B model trained on your own data, DPO-aligned on your own preferences, quantized, speculative-decoded, and served at measurable $/1M tokens. The 2026 open stack is Axolotl v…"
},
{
"name": "Production RAG Chatbot (Regulated Vertical)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P5 P7 P11 P12 P17 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/08-production-rag-chatbot/",
"summary": "Harvey, Glean, Mendable, and LlamaCloud all run the same production shape in 2026. Ingest with docling or Unstructured and ColPali for visuals. Hybrid search. Re-rank with bge-r…"
},
{
"name": "Code Migration Agent (Repo-Level Upgrade)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P5 P7 P11 P13 P14 P15 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/09-code-migration-agent/",
"summary": "Amazon's MigrationBench (Java 8 to 17) and Google's App Engine Py2-to-Py3 migrator set the 2026 bar. Moderne's OpenRewrite does deterministic AST rewrites at scale. Grit targets…"
},
{
"name": "Multi-Agent Software Engineering Team",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P11 P13 P14 P15 P16 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/10-multi-agent-software-team/",
"summary": "SWE-AF's factory architecture, MetaGPT's role-based prompting, AutoGen 0.4's typed actor graph, Cognition's Devin, and Factory's Droids all converged on the same 2026 shape: an …"
},
{
"name": "LLM Observability & Eval Dashboard",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P11 P13 P17 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/11-llm-observability-dashboard/",
"summary": "Langfuse went open-core. Arize Phoenix published the 2026 GenAI semconv mappings. Helicone and Braintrust both doubled down on per-user cost attribution. Traceloop's OpenLLMetry…"
},
{
"name": "Video Understanding Pipeline (Scene → QA)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P4 P6 P7 P11 P12 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/12-video-understanding-pipeline/",
"summary": "Twelve Labs productized Marengo + Pegasus. VideoDB shipped the CRUD-for-video API. AI2's Molmo 2 published open VLM checkpoints. Gemini long-context handles hours of video nativ…"
},
{
"name": "MCP Server with Registry and Governance",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P11 P13 P14 P17 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/13-mcp-server-with-registry/",
"summary": "The Model Context Protocol stopped being the future and became the default tool-use spec in 2026. Anthropic, OpenAI, Google, and every major IDE ship MCP clients. Pinterest publ…"
},
{
"name": "Speculative-Decoding Inference Server",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P3 P7 P10 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/14-speculative-decoding-server/",
"summary": "EAGLE-3 in vLLM 0.7 ships 2.5-3x throughput on real traffic. P-EAGLE (AWS 2026) pushed parallel speculation even further. SGLang's SpecForge trained draft heads at scale. Red Ha…"
},
{
"name": "Constitutional Safety Harness + Red-Team Range",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P10 P11 P13 P14 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/15-constitutional-safety-harness/",
"summary": "Anthropic's Constitutional Classifiers, Meta's Llama Guard 4, Google's ShieldGemma-2, NVIDIA's Nemotron 3 Content Safety, and X-Guard for multilingual coverage defined the 2026 …"
},
{
"name": "GitHub Issue-to-PR Autonomous Agent",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P11 P13 P14 P15 P17",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/16-github-issue-to-pr-agent/",
"summary": "AWS Remote SWE Agents, Cursor Background Agents, OpenAI Codex cloud, and Google Jules all ship the same 2026 product shape: label an issue, get a PR. Run an agent in a cloud san…"
},
{
"name": "Personal AI Tutor (Adaptive, Multimodal)",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "P5 P6 P11 P12 P14 P17 P18",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/17-personal-ai-tutor/",
"summary": "Khanmigo (Khan Academy), Duolingo Max, Google LearnLM / Gemini for Education, Quizlet Q-Chat, and Synthesis Tutor all shipped adaptive multimodal tutoring at scale in 2026. The …"
},
{
"name": "Agent Harness Loop Contract",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/20-agent-harness-loop-contract/",
"summary": "The harness is the agent. The model is a coprocessor. This lesson freezes the loop contract you can wire any model into."
},
{
"name": "Tool Registry with Schema Validation",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/21-tool-registry-schema-validation/",
"summary": "A tool the agent cannot validate is a tool the agent cannot call. Build the registry and the schema checker before you build the tools."
},
{
"name": "JSON-RPC 2.0 Over Newline-Delimited Stdio",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/22-jsonrpc-stdio-transport/",
"summary": "The transport between a model client and a tool server is JSON-RPC over stdio. Hand-rolling it once teaches you what every framing layer is paying for."
},
{
"name": "Function Call Dispatcher",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/23-function-call-dispatcher/",
"summary": "The dispatcher is where the harness pays for every promise the schema made. Timeouts, retries, dedupe, error mapping. All on one seam."
},
{
"name": "Plan-Execute Control Flow",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/24-plan-execute-control-flow/",
"summary": "A plan that cannot survive a failure is a script. A script that can replan is an agent. Build the replanner first."
},
{
"name": "Verification Gates and Observation Budget",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/25-verification-gates-observation-budget/",
"summary": "An agent harness without a verification layer is a wish in a trenchcoat. This lesson builds the deterministic gate chain that decides whether a tool call is allowed to fire, how…"
},
{
"name": "Sandbox Runner with Denylist and Path Jail",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/26-sandbox-runner-denylist/",
"summary": "The verification gate decides whether a tool call should run. The sandbox decides what happens when it does. This lesson ships a subprocess runner that refuses dangerous executa…"
},
{
"name": "Eval Harness with Fixture Tasks",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/27-eval-harness-fixture-tasks/",
"summary": "A coding agent is only as good as the suite of tasks you measure it against. This lesson builds an evaluation harness that takes a folder of fixture tasks, runs each through a c…"
},
{
"name": "Observability with OTel GenAI Spans and Prometheus Metrics",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/28-observability-otel-traces/",
"summary": "An agent harness without observability is a black box that costs money. This lesson hand-rolls a span builder that emits records compliant with the OpenTelemetry GenAI semantic …"
},
{
"name": "End-to-End Coding Agent on the Harness",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "A. Agent harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/29-end-to-end-coding-task-demo/",
"summary": "Track A's payoff. This lesson stitches the gate chain, the sandbox, the eval harness, and the OTel spans into one working coding agent that fixes a real (small, fixture-scale) b…"
},
{
"name": "BPE Tokenizer From Scratch",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/30-bpe-tokenizer-from-scratch/",
"summary": "Bytes in, ids out, ids back to the same bytes. Build the tokenizer that every modern text model still starts from."
},
{
"name": "Tokenized Dataset with Sliding Window",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/31-tokenized-dataset-sliding-window/",
"summary": "A pretraining run is a function from token ids to gradients. This lesson builds the conveyor that feeds the ids in."
},
{
"name": "Token and Positional Embeddings",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/32-token-positional-embeddings/",
"summary": "Ids are integers. The model wants vectors. Two lookup tables sit between them, and the choice of the positional one shapes what the model can learn."
},
{
"name": "Multi-Head Self-Attention",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/33-multihead-self-attention/",
"summary": "One linear projection, three views, H parallel heads, one mask. The attention block as the model actually uses it."
},
{
"name": "Transformer Block from Scratch",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/34-transformer-block/",
"summary": "One block is the unit of every modern decoder LLM. Layer norm, multi head attention, residual, MLP, residual. The pre-LN variant trains stably without warmup. The post-LN varian…",
"keywords": "Causal multi head attention · The MLP · Residual connections do two things"
},
{
"name": "GPT Model Assembly",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/35-gpt-model-assembly/",
"summary": "Twelve blocks stacked, a token embedding, a learned position embedding, a final LayerNorm, and a tied language model head. That is the entire 124 million parameter GPT model. Th…",
"keywords": "Weight tying · Position embedding is learned, not sinusoidal · Generation: temperature, top-k, multinomial"
},
{
"name": "Training Loop and Evaluation",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/36-training-loop-eval/",
"summary": "A loop that does not measure is a loop that lies. This lesson builds the training loop that drives the GPT model: AdamW with weight decay split, a warmup plus cosine learning ra…",
"keywords": "Loss alignment · AdamW decay split · Warmup plus cosine schedule · Held out evaluation · Qualitative sampling as an early signal"
},
{
"name": "Loading Pretrained Weights",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/37-loading-pretrained-weights/",
"summary": "Training a 124 million parameter model from scratch is a budget decision; loading a published checkpoint is a Tuesday. This lesson loads pretrained GPT-2 style weights from a sa…",
"keywords": "The GPT-2 naming convention · The local naming convention · The stub fixture"
},
{
"name": "Classifier Fine-Tuning by Head Swap",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/38-classifier-finetuning/",
"summary": "Track B's first capstone. A pretrained language model is a stack of self-attention blocks ending in a token-prediction head. When you want spam vs ham, the head is wrong but the…"
},
{
"name": "Instruction Tuning by Supervised Fine-Tuning",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/39-instruction-tuning-sft/",
"summary": "A pretrained base model can extend a sequence but cannot follow an instruction. Supervised fine-tuning is the smallest change that fixes this: feed the model paired examples of …"
},
{
"name": "Direct Preference Optimization from Scratch",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/40-dpo-from-scratch/",
"summary": "Reward models and PPO are the classical RLHF stack. DPO collapses that stack into a single supervised loss that fits a policy directly against preference pairs. This lesson deri…"
},
{
"name": "Full Evaluation Pipeline",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "B. NLP LLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/41-eval-pipeline/",
"summary": "Training is the part you can monitor with loss curves. Evaluation is the part you have to design. This lesson builds a unified eval pipeline that takes any trained language mode…"
},
{
"name": "Large Corpus Downloader",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/42-large-corpus-downloader/",
"summary": "Training a language model begins long before the first forward pass. The corpus has to land on disk, decompressed, deduplicated, and addressable, with the resume story already w…",
"keywords": "Streaming with `urllib` · Resume with `Range` · MinHash plus LSH · Shard manifest as a contract"
},
{
"name": "HDF5 Tokenized Corpus",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/43-hdf5-tokenized-corpus/",
"summary": "The downloaded corpus has to land in a layout the trainer can stream from at line speed. JSONL on disk does not survive 16 dataloader workers. HDF5 with a resizable, chunked int…",
"keywords": "Resizable HDF5 done right · Sharded write · Memory-mapped read · Sliding-window dataloader"
},
{
"name": "Cosine LR with Linear Warmup",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/44-cosine-lr-warmup/",
"summary": "The learning-rate schedule is the second most important decision after the loss function. AdamW with a cosine decay and a linear warmup is the modern default for language-model …",
"keywords": "Warmup formula · Cosine formula · Floor after total steps · Gradient norm logging alongside the rate"
},
{
"name": "Gradient Clipping and Mixed Precision",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/45-gradient-clipping-amp/",
"summary": "The optimizer and schedule from the previous lesson assume gradients are sane. They usually are not. A single bad batch can spike the gradient norm by three orders of magnitude.…",
"keywords": "Global L2 norm · autocast and GradScaler · NaN and Inf detection · Scaling factor diagnostics"
},
{
"name": "Gradient Accumulation",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/46-gradient-accumulation/",
"summary": "Train at an effective batch you cannot afford, one micro-batch at a time. Scale the loss, hold the optimizer step, and let the gradients pile up.",
"keywords": "The equivalence proof in code · Where the cost goes · Step 1: equivalence check · Step 2: sync-on-last-step pattern · Step 3: the throughput curve"
},
{
"name": "Checkpoint Save and Resume",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/47-checkpoint-save-resume/",
"summary": "Train interrupts kill runs; checkpoints let them continue. Save model, optimizer, scheduler, loss history, step counter, and RNG state, atomically, so a kill at any moment leave…",
"keywords": "The five state buckets · Atomic save · Sharded checkpoints · Resume continues mid epoch · Step 1: capture and restore RNG state · Step 2: atomic save · Step 3: full checkpoint round trip · Step 4: sharded variant · Step 5: resume demo"
},
{
"name": "Distributed Data Parallel and FSDP from Scratch",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/48-distributed-fsdp-ddp/",
"summary": "Multi-rank training is two collectives and one rule. Broadcast the parameters at startup, average the gradients after backward, never let the ranks disagree about what step they…",
"keywords": "The two collectives that matter · Gradient averaging matches single-process gradient · FSDP sketch · CPU and the gloo backend · Step 1: bring up the process group · Step 2: broadcast at construction · Step 3: all-reduce gradients after backward · Step 4: prove the equivalence · Step 5: FSDP round trip"
},
{
"name": "Language Model Evaluation Harness",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "C. Train end-to-end",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/49-lm-eval-harness/",
"summary": "A model that does well on a task you cannot define is a model that does well by accident. The harness is the task definition, the metric, the runner, and the leaderboard, in one…",
"keywords": "Task spec · The five fixture tasks · The metric contract · The model adapter · The runner · Step 1: seed fixture tasks · Step 2: load tasks · Step 3: implement metrics · Step 4: write the runner · Step 5: emit JSON"
},
{
"name": "Hypothesis Generator",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/50-hypothesis-generator/",
"summary": "A research agent that asks the same question twice is wasting tokens. The trick is forcing each draft to land somewhere new."
},
{
"name": "Literature Retrieval",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/51-literature-retrieval/",
"summary": "A hypothesis is cheap. Knowing whether someone already proved it is the expensive part. Build the retrieval layer that answers that question before the runner spins up a sandbox."
},
{
"name": "Experiment Runner",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/52-experiment-runner/",
"summary": "The loop is only as honest as its measurements. Build the runner that takes a spec, executes it in a sandboxed subprocess, and emits a json metrics blob the evaluator can trust."
},
{
"name": "Result Evaluator",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/53-result-evaluator/",
"summary": "The runner produced numbers. The evaluator decides whether those numbers are an improvement, a regression, or noise. Build the verdict path that turns metrics into a one line co…"
},
{
"name": "Paper Writer",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/54-paper-writer/",
"summary": "A LaTeX skeleton is a contract between the researcher and the typesetter. If the contract is broken the document does not compile, and the failure is loud. Build the skeleton fi…"
},
{
"name": "Critic Loop",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/55-critic-loop/",
"summary": "A critic that returns \"looks good\" the first time is broken. A critic that always returns \"needs work\" is broken. The interesting critic is the one that converges, and you have …"
},
{
"name": "Iteration Scheduler",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/56-iteration-scheduler/",
"summary": "A research loop without a scheduler is a queue with delusions. The scheduler is where the loop decides what to stop exploring, and that decision is the whole game."
},
{
"name": "End-to-End Research Demo",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "D. Auto research",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/57-end-to-end-research-demo/",
"summary": "A demo is the place where every contract you wrote earlier has to compose. If any one of them leaks, the demo is the lesson that catches it."
},
{
"name": "Vision Encoder Patches",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "E. Multimodal VLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/58-vision-encoder-patches/",
"summary": "A vision model that reads pixels needs a tokenizer for pixels. Patch embedding is that tokenizer. Cut the image into a grid of squares, flatten each square, project it through o…",
"keywords": "Why patches, not pixels · Why a linear projection is enough · The `Conv2d` trick · Position embeddings · Equivalence as a sanity check"
},
{
"name": "Vision Transformer Encoder",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "E. Multimodal VLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/59-vit-transformer/",
"summary": "Patches alone do not see. A 12-layer pre-LN transformer with 12 attention heads turns the sequence of patch tokens into a sequence of contextual tokens, with the CLS token pooli…",
"keywords": "Pre-LN vs post-LN · Multi-head self-attention · Why the 4x feed-forward expansion · Causal mask or not? · What the CLS token learns"
},
{
"name": "Projection Layer for Modality Alignment",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "E. Multimodal VLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/60-projection-layer-modality-align/",
"summary": "A vision encoder produces image tokens. A text decoder consumes text tokens. The two live in different vector spaces. A small two-layer MLP projects image tokens into the text e…",
"keywords": "Pooling before projection · Why two layers and not one · Cosine alignment loss · Frozen encoder is the trick"
},
{
"name": "Cross-Attention Fusion",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "E. Multimodal VLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/61-cross-attention-fusion/",
"summary": "The projection layer aligns one image vector with one caption vector. A real vision-language decoder needs every text token to attend to every patch token, so the model can grou…",
"keywords": "Mask shapes · Why no mask on cross-attention · Key/value caching · Block composition"
},
{
"name": "Vision-Language Pretraining",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "E. Multimodal VLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/62-vision-language-pretraining/",
"summary": "The encoder, projection, and decoder are wired. Now train them together. Two objectives drive learning: a contrastive image-text loss (InfoNCE) that pulls matching pairs togethe…",
"keywords": "InfoNCE in one paragraph · Temperature matters · Language modeling loss · Combining the losses · Why 50 steps is enough for a demo"
},
{
"name": "Multimodal Evaluation",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "E. Multimodal VLM",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/63-multimodal-eval/",
"summary": "Training is half the loop. The other half is measurement. This lesson builds three evaluation surfaces from primitives: image-caption retrieval reported as R@1, R@5, R@10; visua…",
"keywords": "Recall@K from a similarity matrix · VQA exact match · BLEU-4 · Synthetic eval suite"
},
{
"name": "Chunking Strategies, Compared",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "F. Advanced RAG",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/64-chunking-strategies-advanced/",
"summary": "Chunking decides what your retriever can ever surface. Get the boundaries wrong and no embedding model, no reranker, no LLM can repair the damage downstream.",
"keywords": "Fixed-window · Sentence · Recursive split · Semantic clustering · Structural markdown headers · How recall@k measures the boundary choice"
},
{
"name": "Hybrid Retrieval with BM25 and Dense Embeddings",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "F. Advanced RAG",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/65-hybrid-retrieval-bm25-dense/",
"summary": "Lexical and semantic retrieval fail on opposite query distributions. Hybrid retrieval with reciprocal rank fusion does not interpolate, it votes - and the vote wins on every que…",
"keywords": "BM25 in one paragraph · Dense retrieval in one paragraph · Reciprocal rank fusion, the published formula · Why fusion beats score-weighted interpolation"
},
{
"name": "Cross-Encoder Reranker",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "F. Advanced RAG",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/66-reranker-cross-encoder/",
"summary": "A bi-encoder embeds query and document independently. A cross-encoder concatenates them and reads both at once. The cross-encoder is the smartest reader and the slowest. Used as…",
"keywords": "The cross-encoder's input shape · Why this lesson trains a tiny one · Latency vs quality"
},
{
"name": "Query Rewriting: HyDE, Multi-Query, and Decomposition",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "F. Advanced RAG",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/67-query-rewriting-hyde/",
"summary": "The query the user types is not the query your retriever wants. Rewriting bridges the gap before retrieval, so the index sees something closer to what the answer looks like.",
"keywords": "HyDE in detail · Multi-query expansion in detail · Decomposition in detail · Why all three exist"
},
{
"name": "RAG Evaluation: Precision, Recall, MRR, nDCG, Faithfulness, Answer Relevance",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "F. Advanced RAG",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/68-rag-eval-precision-recall/",
"summary": "If you cannot grade your retrieval and your answer at the same time, you cannot ship the system. The two are not the same metric and the same prompt fails on different axes.",
"keywords": "Precision@k · Recall@k · MRR (Mean Reciprocal Rank) · nDCG@k · Faithfulness · Answer relevance"
},
{
"name": "End-to-End RAG System",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "F. Advanced RAG",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/69-end-to-end-rag-system/",
"summary": "Six lessons of components. One pipeline. One eval loop. One self-terminating demo. This is the system you ship.",
"keywords": "Wiring choices · Answer generator with citations · The self-terminating demo"
},
{
"name": "Task Spec Format",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "G. Eval framework",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/70-task-spec-format/",
"summary": "An eval harness is only as good as the contract its tasks honour. Freeze the JSONL shape and the metric vocabulary before you write a single scoring function."
},
{
"name": "Classical Metrics",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "G. Eval framework",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/71-classical-metrics/",
"summary": "BLEU, ROUGE-L, F1, exact-match, accuracy. Five metrics that still account for most published LLM eval numbers. Implement each from first principles so you know what the number m…"
},
{
"name": "Code Exec Metric",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "G. Eval framework",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/72-code-exec-metric/",
"summary": "Generated code is right when it passes the tests. The eval harness has to extract code, run it without crashing the host, and tally pass-rates honestly. This lesson builds that …"
},
{
"name": "Perplexity and Calibration",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "G. Eval framework",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/73-perplexity-calibration/",
"summary": "If your model says 90 percent confident on a thousand answers and gets six hundred right, it is not well calibrated. Calibration is half of trustworthy eval. The other half is p…"
},
{
"name": "Leaderboard Aggregation",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "G. Eval framework",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/74-leaderboard-aggregation/",
"summary": "Per-task scores are easy. Per-model rankings across heterogeneous tasks are harder. Statistical significance on a thousand-prediction leaderboard is the part everyone skips. Thi…"
},
{
"name": "End-to-End Eval Runner",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "G. Eval framework",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/75-end-to-end-eval-runner/",
"summary": "Five lessons of plumbing, one lesson to glue them. The runner reads the task spec from lesson 70, calls a model through an adapter, scores with lessons 71 and 72, attaches the c…"
},
{
"name": "Collective Ops From Scratch",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "H. Distributed train",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/76-collective-ops-from-scratch/",
"summary": "The four collective operations that hold distributed training together are allreduce, broadcast, allgather, and reduce_scatter. Every other primitive a training framework offers…",
"keywords": "Ring allreduce in two passes · Queue mesh as a stand-in for NCCL · Verify against gloo"
},
{
"name": "Data Parallel DDP From Scratch",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "H. Distributed train",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/77-data-parallel-ddp/",
"summary": "DistributedDataParallel is a hook on top of allreduce. Wrap a model, broadcast the initial parameters from rank 0 so every rank starts identical, install a backward hook on ever…",
"keywords": "The three operations DDP needs · Why mean and not sum · Why bucket gradients · Why pin the seed"
},
{
"name": "ZeRO Optimizer State Sharding",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "H. Distributed train",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/78-zero-parameter-sharding/",
"summary": "Adam stores two moment estimates per parameter, both in float32. A 7B-parameter model carries 56 GB of optimiser state. ZeRO stage 1 shards that across N ranks; each rank owns 1…",
"keywords": "Stages of ZeRO · The memory math, real numbers · Why reduce_scatter beats allreduce-then-shard"
},
{
"name": "Pipeline Parallel and Bubble Analysis",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "H. Distributed train",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/79-pipeline-parallel/",
"summary": "Tensor parallelism splits the matrix multiply across ranks. Pipeline parallelism splits the model across ranks, one stage per rank. Microbatches flow through the pipeline. The e…",
"keywords": "GPipe schedule · 1F1B schedule · Why equal compute per stage matters · Microbatch versus batch"
},
{
"name": "Sharded Checkpoint and Atomic Resume",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "H. Distributed train",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/80-checkpoint-sharded-resume/",
"summary": "A 70B-parameter training job is paused by a node failure every few hours. The checkpoint format decides whether you lose 30 minutes or 30 hours. A sharded checkpoint writes ever…",
"keywords": "Manifest schema · Atomic write · Three failure modes the schema must defend against · Why per-rank files, not one big file"
},
{
"name": "End-to-End Distributed Training",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "H. Distributed train",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/81-end-to-end-distributed-train/",
"summary": "Lessons 76 through 80 each built one piece. This is the assembly: a tiny GPT trained across 4 simulated ranks with DDP for gradient sync, ZeRO-1 for optimiser-state sharding, an…",
"keywords": "The mini GPT · The composition rules · Why a tiny GPT and not just an MLP · Self-terminating means exit 0"
},
{
"name": "Jailbreak Taxonomy",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "I. Safety harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/82-jailbreak-taxonomy/",
"summary": "A safety harness without a taxonomy is a coin flip. Name the attack before you defend it."
},
{
"name": "Prompt Injection Detector",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "I. Safety harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/83-prompt-injection-detector/",
"summary": "A detector is a function from prompt to confidence and category. Anything else is a vibe."
},
{
"name": "Refusal Evaluation",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "I. Safety harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/84-refusal-evaluation/",
"summary": "Helpfulness on benign prompts and refusal on harmful prompts are two metrics, not one. Measure both."
},
{
"name": "Content Classifier Integration",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "I. Safety harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/85-content-classifier-integration/",
"summary": "Classifiers on the output side answer a different question than rules on the input side. Both need a policy router."
},
{
"name": "Constitutional Rules Engine",
"status": "complete",
"type": "Capstone",
"lang": "Python, YAML",
"combines": "I. Safety harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/86-constitutional-rules-engine/",
"summary": "A rule is a name, a predicate, and an explanation. Anything missing one of those three is a vibe, not a rule."
},
{
"name": "End-to-End Safety Gate",
"status": "complete",
"type": "Capstone",
"lang": "Python",
"combines": "I. Safety harness",
"url": "https://github.com/rohitg00/ai-engineering-from-scratch/tree/main/phases/19-capstone-projects/87-end-to-end-safety-gate/",
"summary": "Pre-gen, during-gen, post-gen. Three checkpoints, one verdict, an audit trail per request."
}
]
}
];
const GLOSSARY = [
{
"term": "Agent",
"says": "An autonomous AI that thinks and acts on its own",
"means": "A while loop where an LLM decides what tool to call next, executes it, sees the result, and repeats"
},
{
"term": "Attention",
"says": "How the AI focuses on important parts",
"means": "A mechanism where every token computes a weighted sum of all other tokens' values, with weights determined by how relevant they are (via dot product of query and key vectors)"
},
{
"term": "Alignment",
"says": "Making AI safe",
"means": "The technical challenge of making an AI system's behavior match human intentions, values, and preferences, including edge cases the designer didn't anticipate"
},
{
"term": "Autoregressive",
"says": "The AI generates one word at a time",
"means": "A model that predicts the next token conditioned on all previous tokens, then feeds that prediction back as input for the next step. GPT, LLaMA, and Claude are all autoregressive."
},
{
"term": "Activation Function",
"says": "The nonlinear thing between layers",
"means": "A function applied after each linear layer that introduces nonlinearity. Without it, stacking any number of linear layers collapses to a single linear transformation. ReLU, GELU, and SiLU are the most common. The choice directly affects whether gradients flow during training."
},
{
"term": "Adam (Optimizer)",
"says": "The default optimizer",
"means": "Adaptive Moment Estimation. Combines momentum (first moment) with adaptive learning rates per parameter (second moment). Has bias correction for early steps. Works well across most tasks without much tuning."
},
{
"term": "AdamW",
"says": "Adam but better",
"means": "Adam with decoupled weight decay. In standard Adam, L2 regularization gets scaled by the adaptive learning rate per parameter, which is not what you want. AdamW applies weight decay directly to the weights, independent of the gradient statistics. The default optimizer for training transformers."
},
{
"term": "Autograd",
"says": "Automatic gradients",
"means": "A system that records operations on tensors and automatically computes gradients via reverse-mode differentiation. PyTorch's autograd builds a computation graph on-the-fly (dynamic graph), while JAX uses function transformations (grad). This is what makes backpropagation practical -- you write the forward pass, and the framework computes all the derivatives."
},
{
"term": "Batch Size",
"says": "How many examples at once",
"means": "The number of training examples processed in one forward/backward pass before updating weights. Larger batches give more stable gradient estimates but use more memory. Typical values: 32-512 for training, larger for inference. Batch size interacts with learning rate -- double the batch, double the LR (linear scaling rule)."
},
{
"term": "Backpropagation",
"says": "How neural networks learn",
"means": "An algorithm that computes how much each weight contributed to the error by applying the chain rule backward through the network, then adjusts weights proportionally"
},
{
"term": "Context Window",
"says": "How much the AI can remember",
"means": "The maximum number of tokens (input + output) that fit in a single API call. Not memory — it's a fixed-size buffer that resets every call"
},
{
"term": "Chain of Thought (CoT)",
"says": "Making the AI think step by step",
"means": "A prompting technique where you ask the model to show its reasoning steps, which improves accuracy on multi-step problems because each step conditions the next token generation"
},
{
"term": "CNN (Convolutional Neural Network)",
"says": "Image AI",
"means": "A neural network that uses convolution operations (sliding filters over the input) to detect local patterns. Stacking convolutions detects increasingly complex features: edges, textures, objects."
},
{
"term": "CUDA",
"says": "GPU programming",
"means": "NVIDIA's parallel computing platform. Lets you run matrix operations on thousands of GPU cores simultaneously. PyTorch and TensorFlow use CUDA under the hood."
},
{
"term": "Chunking",
"says": "Splitting documents into pieces",
"means": "Breaking text into segments before embedding for retrieval. Chunk size determines the granularity of search results. Too small: loses context. Too large: dilutes relevance. Common strategies: fixed-size with overlap, sentence-based, or semantic splitting. Typical chunk size: 256-512 tokens with 10-20% overlap."
},
{
"term": "Contrastive Learning",
"says": "Learning by comparison",
"means": "Training by pulling similar pairs closer and pushing dissimilar pairs apart in embedding space. CLIP uses this: matching image-text pairs vs non-matching ones."
},
{
"term": "Cosine Similarity",
"says": "How similar two vectors are",
"means": "The cosine of the angle between two vectors: dot(a, b) / (||a|| * ||b||). Ranges from -1 (opposite) to 1 (identical direction). Ignores magnitude, only cares about direction. The standard similarity metric for embeddings and semantic search."
},
{
"term": "Cross-Entropy",
"says": "The classification loss",
"means": "Measures the difference between two probability distributions. For classification: -sum(y_true * log(y_pred)). For language models: the negative log probability of the correct next token. Lower is better. Perplexity is just exp(cross-entropy)."
},
{
"term": "Data Augmentation",
"says": "Making more training data",
"means": "Creating modified copies of existing data (rotate images, add noise, paraphrase text) to increase training set diversity without collecting new data. Reduces overfitting."
},
{
"term": "Decoder",
"says": "The output part",
"means": "In transformers, a decoder uses causal (masked) self-attention so each position can only attend to earlier positions. GPT is decoder-only. BERT is encoder-only. T5 is encoder-decoder."
},
{
"term": "Diffusion Model",
"says": "AI that generates images from noise",
"means": "A model trained to reverse a gradual noising process — it learns to predict and remove noise, and at generation time starts from pure noise and iteratively denoises"
},
{
"term": "DPO (Direct Preference Optimization)",
"says": "A simpler RLHF",
"means": "A training method that skips the reward model entirely — it directly optimizes the language model to prefer the better response in pairs of human preferences"
},
{
"term": "Dropout",
"says": "Randomly turning off neurons",
"means": "During training, randomly set a fraction of activations to zero. Forces the network to not rely on any single neuron. Turned off during inference. Simple but effective regularization."
},
{
"term": "Eigenvalue",
"says": "Some math thing for PCA",
"means": "For a matrix A, an eigenvalue lambda satisfies Av = lambda*v for some vector v. It tells you how much the matrix scales vectors in that direction. Large eigenvalues = directions of high variance in your data."
},
{
"term": "Embedding",
"says": "Some AI magic that turns words into numbers",
"means": "A learned mapping from discrete items (words, images, users) to dense vectors in continuous space, where similar items end up close together"
},
{
"term": "Encoder",
"says": "The input part",
"means": "In transformers, an encoder uses bidirectional self-attention so each position can attend to all positions. BERT is encoder-only. Good for understanding tasks (classification, NER) but not generation."
},
{
"term": "Epoch",
"says": "One pass through the data",
"means": "Exactly that. One complete pass through every example in the training set. Multiple epochs = seeing the data multiple times. More epochs can improve learning but risks overfitting."
},
{
"term": "Feature",
"says": "A column in your data",
"means": "An individual measurable property of the data. In classical ML, you engineer features by hand. In deep learning, the network learns features automatically from raw data."
},
{
"term": "Few-Shot",
"says": "Give the AI some examples first",
"means": "Including a small number of input-output examples in the prompt before asking the model to perform a task. Typically 3-5 examples. The model pattern-matches on these examples to understand the desired format and behavior. Contrast with zero-shot (no examples) and fine-tuning (thousands of examples baked into weights)."
},
{
"term": "Fine-tuning",
"says": "Training the AI on your data",
"means": "Starting with a pre-trained model's weights and continuing training on a smaller, task-specific dataset. Only updates existing weights, doesn't add new knowledge from scratch"
},
{
"term": "Function Calling",
"says": "AI that can use tools",
"means": "A structured way for LLMs to request execution of external functions. You define tools with JSON Schema descriptions, the model outputs a structured JSON object specifying which function to call with what arguments, your code executes it, and the result goes back to the model. Not the same as agents -- function calling is the mechanism, agents are the loop."
},
{
"term": "Guardrails",
"says": "Safety filters for AI",
"means": "Input/output validation layers around an LLM that detect and block harmful content, prompt injection attempts, PII leakage, or off-topic responses. Typically a pipeline: input filter -> LLM -> output filter. Can be rule-based (regex, keyword lists) or model-based (classifier that scores safety)."
},
{
"term": "GPT",
"says": "ChatGPT\" or \"The AI",
"means": "Generative Pre-trained Transformer — a specific architecture that predicts the next token using a decoder-only transformer trained on large text corpora"
},
{
"term": "GAN (Generative Adversarial Network)",
"says": "Two AIs fighting each other",
"means": "A generator network tries to create realistic data while a discriminator network tries to tell real from fake. They train together: the generator gets better at fooling the discriminator, and the discriminator gets better at detecting fakes."
},
{
"term": "Gradient",
"says": "The slope",
"means": "A vector of partial derivatives pointing in the direction of steepest increase. In ML, you go opposite to the gradient (gradient descent) to minimize the loss."
},
{
"term": "Gradient Descent",
"says": "How AI improves",
"means": "An optimization algorithm that adjusts parameters in the direction that reduces the loss function most steeply, like walking downhill in a high-dimensional landscape"
},
{
"term": "Hyperparameter",
"says": "Settings you tune",
"means": "Values set before training that control the training process itself: learning rate, batch size, number of layers, dropout rate. Unlike model parameters (weights), these aren't learned from data."
},
{
"term": "Hallucination",
"says": "The AI is lying\" or \"making things up",
"means": "The model generates plausible-sounding text that isn't grounded in its training data or the given context — it's pattern-completing, not fact-retrieving"
},
{
"term": "Inference",
"says": "Running the AI",
"means": "Using a trained model to make predictions on new data. No weight updates happen. This is what you do in production: send input, get output."
},
{
"term": "Inductive Bias",
"says": "Never heard of it",
"means": "The assumptions built into a model's architecture. CNNs assume local patterns matter (convolution). RNNs assume order matters (sequential processing). Transformers assume everything might relate to everything (attention). The right bias helps the model learn faster from less data."
},
{
"term": "JAX",
"says": "Google's ML framework",
"means": "A NumPy-compatible library that adds automatic differentiation (grad), JIT compilation (jit), automatic vectorization (vmap), and multi-device parallelism (pmap). Unlike PyTorch's object-oriented style, JAX is purely functional -- no hidden state, no in-place mutation. Used by Google DeepMind for AlphaFold, Gemini, and large-scale research."
},
{
"term": "KV Cache",
"says": "Makes inference faster",
"means": "During autoregressive generation, caching the key and value matrices from previous tokens so you don't recompute them at each step. Trades memory for speed. Essential for fast LLM inference."
},
{
"term": "Latent Space",
"says": "The hidden representation",
"means": "A compressed, learned representation space where similar inputs map to nearby points. Autoencoders, VAEs, and diffusion models all work in latent space. It's lower-dimensional than the input but captures the important structure."
},
{
"term": "Learning Rate",
"says": "How fast the AI learns",
"means": "A scalar that controls step size during gradient descent. Too high: overshoots the minimum and diverges. Too low: converges too slowly or gets stuck. The single most important hyperparameter."
},
{
"term": "LLM (Large Language Model)",
"says": "AI\" or \"the brain",
"means": "A transformer-based neural network trained to predict the next token in a sequence, with billions of parameters, trained on internet-scale text data"
},
{
"term": "LoRA (Low-Rank Adaptation)",
"says": "Efficient fine-tuning",
"means": "Instead of updating all weights, insert small low-rank matrices alongside the original weights. Only these small matrices are trained, reducing memory by 10-100x"
},
{
"term": "Loss Function",
"says": "How wrong the AI is",
"means": "A function that measures the gap between predicted and actual output. Training minimizes this function. MSE for regression, cross-entropy for classification, contrastive loss for embeddings. The choice of loss function defines what \"good\" means to the model."
},
{
"term": "Mixed Precision",
"says": "Training trick for speed",
"means": "Using float16 for forward pass and most operations (faster, less memory) but keeping float32 for gradient accumulation and weight updates (more precise). Gets 2x speedup with negligible accuracy loss."
},
{
"term": "MoE (Mixture of Experts)",
"says": "Only part of the model runs",
"means": "A model with many \"expert\" subnetworks where a routing mechanism sends each input to only a few experts. The full model is huge but each forward pass is cheap because most experts are skipped. Mixtral and GPT-4 use this."
},
{
"term": "MCP (Model Context Protocol)",
"says": "A way for AI to use tools",
"means": "An open protocol (JSON-RPC over stdio/HTTP) that standardizes how AI applications connect to external data sources and tools, with typed schemas for tools, resources, and prompts"
},
{
"term": "NaN (Not a Number)",
"says": "Training crashed",
"means": "A floating-point value indicating undefined results (0/0, inf-inf). In training, NaN loss usually means: learning rate too high, exploding gradients, log of zero, or division by zero. Always the first thing to check when training fails."
},
{
"term": "Normalization",
"says": "Scaling the data",
"means": "Adjusting values to a standard range. Batch normalization normalizes across a batch. Layer normalization normalizes across features. Both stabilize training and allow higher learning rates."
},
{
"term": "Overfitting",
"says": "The model memorized the data",
"means": "The model performs well on training data but poorly on unseen data. It learned the noise, not the signal. Fix with: more data, regularization (dropout, weight decay), early stopping, data augmentation, simpler model."
},
{
"term": "Optimizer",
"says": "The thing that updates weights",
"means": "An algorithm that uses gradients to update model parameters. SGD is the simplest. Adam is the most common. Each optimizer has different properties: convergence speed, memory usage, sensitivity to hyperparameters."
},
{
"term": "Parameter",
"says": "Model size",
"means": "A learnable value in the model, typically a weight or bias. \"7B parameters\" means 7 billion learnable numbers. Each float32 parameter takes 4 bytes, so 7B parameters = 28GB of memory just for the weights."
},
{
"term": "Perplexity",
"says": "How confused the model is",
"means": "The exponential of the average cross-entropy loss. Lower is better. A perplexity of 10 means the model is as uncertain as if it were choosing uniformly among 10 tokens at each step."
},
{
"term": "Precision & Recall",
"says": "Accuracy metrics",
"means": "Precision = of items you flagged, how many were correct. Recall = of all correct items, how many did you find. They trade off: catching every spam email (high recall) means more false alarms (low precision). F1 score is their harmonic mean. Use precision when false positives are costly, recall when false negatives are costly."
},
{
"term": "Prompt Engineering",
"says": "Talking to AI the right way",
"means": "Designing the input text to reliably produce desired outputs -- including system prompts, few-shot examples, format instructions, and chain-of-thought triggers"
},
{
"term": "Prompt Injection",
"says": "Hacking the AI with words",
"means": "An attack where malicious text in the input overrides the system prompt or instructions. Direct injection: user types \"Ignore previous instructions.\" Indirect injection: a retrieved document contains hidden instructions. The LLM equivalent of SQL injection. No complete solution exists -- defense is layers of input validation, output filtering, and privilege separation."
},
{
"term": "QLoRA",
"says": "LoRA but cheaper",
"means": "Quantized LoRA. Keeps the frozen base model weights in 4-bit precision (NF4 format) while training LoRA adapters in 16-bit. Reduces memory by another 3-4x compared to standard LoRA. A 7B model that needs 14GB with LoRA fits in 4-6GB with QLoRA. Quality is within 1% of full fine-tuning on most benchmarks."
},
{
"term": "RAG (Retrieval-Augmented Generation)",
"says": "AI that can search",
"means": "A pattern where you retrieve relevant documents from a knowledge base (using embedding similarity), stuff them into the prompt, and let the LLM answer based on that context"
},
{
"term": "RLHF (Reinforcement Learning from Human Feedback)",
"says": "How they make AI helpful",
"means": "A training pipeline: (1) collect human preferences on model outputs, (2) train a reward model on those preferences, (3) use PPO to optimize the LLM to produce higher-reward outputs"
},
{
"term": "Quantization",
"says": "Making the model smaller",
"means": "Reducing the precision of model weights from float32 (4 bytes) to int8 (1 byte) or int4 (0.5 bytes). Trades a small amount of accuracy for 4-8x less memory and faster inference. GPTQ, AWQ, and GGUF are common formats."
},
{
"term": "ReLU",
"says": "Activation function",
"means": "Rectified Linear Unit: f(x) = max(0, x). The simplest non-linear activation. Fast to compute, doesn't saturate for positive values. Used everywhere because it works and is cheap. Variants: LeakyReLU, GELU, SiLU."
},
{
"term": "ROUGE",
"says": "Summarization metric",
"means": "Recall-Oriented Understudy for Gisting Evaluation. Measures overlap between generated text and reference text. ROUGE-1 counts unigram matches, ROUGE-2 counts bigram matches, ROUGE-L finds the longest common subsequence. Cheap to compute but only measures surface similarity -- two sentences with the same meaning but different words score poorly."
},
{
"term": "Semantic Search",
"says": "Smart search that understands meaning",
"means": "Finding documents by meaning rather than keyword matching. Embed the query and all documents into the same vector space, then return documents whose embeddings are closest to the query embedding. \"payment failed\" finds \"transaction declined\" even though they share no words. Powered by embedding models + vector databases."
},
{
"term": "Streaming",
"says": "Seeing the response appear word by word",
"means": "The LLM sends tokens as they are generated rather than waiting for the complete response. Uses Server-Sent Events (SSE) or WebSocket protocols. Reduces perceived latency from seconds to milliseconds for the first token. Essential for production chat interfaces. Each chunk contains a delta (partial token or word)."
},
{
"term": "Self-Attention",
"says": "How the model decides what to focus on",
"means": "Each token computes query, key, and value vectors. Attention weight between two tokens = dot product of their query and key, scaled and softmaxed. Output = weighted sum of value vectors. Lets every token see every other token."
},
{
"term": "SFT (Supervised Fine-Tuning)",
"says": "Teaching the model to follow instructions",
"means": "Fine-tuning a pre-trained model on (instruction, response) pairs. The model learns to generate the response given the instruction. This is what turns a base model into a chat model."
},
{
"term": "Softmax",
"says": "Turns numbers into probabilities",
"means": "softmax(x_i) = exp(x_i) / sum(exp(x_j)). Transforms a vector of arbitrary real numbers into a probability distribution (all positive, sums to 1). Used in classification heads, attention weights, and anywhere you need probabilities."
},
{
"term": "Swarm",
"says": "A bunch of AI agents working together like bees",
"means": "Multiple agents sharing state and coordinating through message passing, with emergent behavior arising from simple individual rules rather than central control"
},
{
"term": "System Prompt",
"says": "The AI's instructions",
"means": "A special message at the start of a conversation that sets the model's behavior, persona, and constraints. Processed before user messages. Not visible to the user in most UIs. Defines what the model should and shouldn't do, its tone, format preferences, and domain focus. Different from user prompts -- system prompts are set by the developer."
},
{
"term": "Tensor",
"says": "A multi-dimensional array",
"means": "The fundamental data structure in deep learning frameworks. A 0D tensor is a scalar, 1D is a vector, 2D is a matrix, 3D+ is a tensor. In PyTorch and JAX, tensors track their computation history for automatic differentiation and can live on CPU or GPU. All neural network inputs, outputs, weights, and gradients are tensors."
},
{
"term": "Token",
"says": "A word",
"means": "A subword unit (typically 3-4 characters in English) produced by a tokenizer like BPE. \"unbelievable\" might be 3 tokens: \"un\" + \"believ\" + \"able\""
},
{
"term": "Temperature",
"says": "Creativity setting",
"means": "A scalar that divides logits before softmax. Temperature=1 is default. Higher = flatter distribution = more random outputs. Lower = sharper distribution = more deterministic. Temperature=0 is argmax (always pick the most likely token)."
},
{
"term": "Transfer Learning",
"says": "Using a pre-trained model",
"means": "Taking a model trained on one task and adapting it to a different task. The early layers learn general features (edges, syntax patterns) that transfer. Only the later layers need task-specific training. This is why you can fine-tune BERT for any NLP task."
},
{
"term": "Transformer",
"says": "The architecture behind modern AI",
"means": "A neural network architecture that processes sequences using self-attention (letting every position attend to every other position) instead of recurrence, enabling massive parallelization"
},
{
"term": "Underfitting",
"says": "The model isn't learning",
"means": "The model is too simple to capture the patterns in the data. Training loss stays high. Fix with: more parameters, more layers, longer training, lower regularization, better features."
},
{
"term": "VAE (Variational Autoencoder)",
"says": "A generative model",
"means": "An autoencoder that learns a smooth latent space by forcing the encoder output to follow a Gaussian distribution. You can sample from this distribution and decode to generate new data. The reparameterization trick makes it trainable via backpropagation."
},
{
"term": "Vector Database",
"says": "A special database for AI",
"means": "A database optimized for storing vectors (dense arrays of floats) and performing fast approximate nearest-neighbor search. The core operation in similarity search, RAG, and recommendation systems."
},
{
"term": "Weight",
"says": "What the model learned",
"means": "A single number in a model's parameter matrix. A linear layer with input size 768 and output size 3072 has 768*3072 = 2,359,296 weights. Training adjusts each weight to minimize the loss function."
},
{
"term": "Weight Decay",
"says": "Regularization",
"means": "Adding a penalty proportional to the magnitude of weights to the loss function. Equivalent to L2 regularization. Prevents weights from growing too large. Typical value: 0.01-0.1."
},
{
"term": "Zero-Shot",
"says": "No training needed",
"means": "Using a model on a task it wasn't explicitly trained for, with no task-specific examples in the prompt. The model generalizes from pre-training. Works because large models have seen enough variety to handle new task formats."
}
];
const ARTIFACTS = [
{
"kind": "prompt",
"name": "prompt-env-check",
"description": "Diagnose and fix AI engineering environment setup issues",
"tags": [],
"phase": 0,
"lesson": 1,
"lessonPath": "phases/00-setup-and-tooling/01-dev-environment",
"file": "phases/00-setup-and-tooling/01-dev-environment/outputs/prompt-env-check.md"
},
{
"kind": "prompt",
"name": "prompt-api-troubleshooter",
"description": "Diagnose and fix common AI API errors (auth, rate limits, timeouts)",
"tags": [],
"phase": 0,
"lesson": 4,
"lessonPath": "phases/00-setup-and-tooling/04-apis-and-keys",
"file": "phases/00-setup-and-tooling/04-apis-and-keys/outputs/prompt-api-troubleshooter.md"
},
{
"kind": "prompt",
"name": "prompt-notebook-helper",
"description": "Debug Jupyter notebook issues including kernel crashes, memory problems, and display failures",
"tags": [],
"phase": 0,
"lesson": 5,
"lessonPath": "phases/00-setup-and-tooling/05-jupyter-notebooks",
"file": "phases/00-setup-and-tooling/05-jupyter-notebooks/outputs/prompt-notebook-helper.md"
},
{
"kind": "prompt",
"name": "prompt-data-helper",
"description": "Find and load the right dataset for an AI/ML task",
"tags": [],
"phase": 0,
"lesson": 9,
"lessonPath": "phases/00-setup-and-tooling/09-data-management",
"file": "phases/00-setup-and-tooling/09-data-management/outputs/prompt-data-helper.md"
},
{
"kind": "prompt",
"name": "prompt-debug-ai-code",
"description": "Diagnose AI-specific bugs including NaN loss, shape errors, training failures, and OOM",
"tags": [],
"phase": 0,
"lesson": 12,
"lessonPath": "phases/00-setup-and-tooling/12-debugging-and-profiling",
"file": "phases/00-setup-and-tooling/12-debugging-and-profiling/outputs/prompt-debug-ai-code.md"
},
{
"kind": "prompt",
"name": "prompt-linear-algebra-tutor",
"description": "Teach linear algebra through geometric intuition and AI applications",
"tags": [],
"phase": 1,
"lesson": 1,
"lessonPath": "phases/01-math-foundations/01-linear-algebra-intuition",
"file": "phases/01-math-foundations/01-linear-algebra-intuition/outputs/prompt-linear-algebra-tutor.md"
},
{
"kind": "prompt",
"name": "prompt-matrix-operations",
"description": "Teaches matrix operations through geometric intuition, connecting abstract math to neural network mechanics",
"tags": [],
"phase": 1,
"lesson": 2,
"lessonPath": "phases/01-math-foundations/02-vectors-matrices-operations",
"file": "phases/01-math-foundations/02-vectors-matrices-operations/outputs/prompt-matrix-operations.md"
},
{
"kind": "prompt",
"name": "prompt-transformation-visualizer",
"description": "Explain what a matrix transformation does geometrically given its entries",
"tags": [],
"phase": 1,
"lesson": 3,
"lessonPath": "phases/01-math-foundations/03-matrix-transformations",
"file": "phases/01-math-foundations/03-matrix-transformations/outputs/prompt-transformation-visualizer.md"
},
{
"kind": "skill",
"name": "skill-gradient-computation",
"description": "Compute gradients of common ML loss functions and choose the right derivative approach",
"tags": [
"calculus",
"gradients",
"backpropagation"
],
"phase": 1,
"lesson": 4,
"lessonPath": "phases/01-math-foundations/04-calculus-for-ml",
"file": "phases/01-math-foundations/04-calculus-for-ml/outputs/skill-gradient-computation.md"
},
{
"kind": "skill",
"name": "skill-autodiff",
"description": "Build, debug, and reason about automatic differentiation systems",
"tags": [],
"phase": 1,
"lesson": 5,
"lessonPath": "phases/01-math-foundations/05-chain-rule-and-autodiff",
"file": "phases/01-math-foundations/05-chain-rule-and-autodiff/outputs/skill-autodiff.md"
},
{
"kind": "skill",
"name": "skill-probability-reasoning",
"description": "Choose the right probability distribution for a given ML problem",
"tags": [
"probability",
"distributions",
"modeling"
],
"phase": 1,
"lesson": 6,
"lessonPath": "phases/01-math-foundations/06-probability-and-distributions",
"file": "phases/01-math-foundations/06-probability-and-distributions/outputs/skill-probability-reasoning.md"
},
{
"kind": "prompt",
"name": "prompt-bayesian-reasoning",
"description": "Walk through Bayesian reasoning step by step for any scenario",
"tags": [],
"phase": 1,
"lesson": 7,
"lessonPath": "phases/01-math-foundations/07-bayes-theorem",
"file": "phases/01-math-foundations/07-bayes-theorem/outputs/prompt-bayesian-reasoning.md"
},
{
"kind": "prompt",
"name": "prompt-optimizer-guide",
"description": "Guides the user through choosing the right optimizer for their specific machine learning problem",
"tags": [],
"phase": 1,
"lesson": 8,
"lessonPath": "phases/01-math-foundations/08-optimization",
"file": "phases/01-math-foundations/08-optimization/outputs/prompt-optimizer-guide.md"
},
{
"kind": "skill",
"name": "skill-information-theory",
"description": "Apply information theory concepts to ML loss functions, model evaluation, and feature selection",
"tags": [
"information-theory",
"entropy",
"loss-functions"
],
"phase": 1,
"lesson": 9,
"lessonPath": "phases/01-math-foundations/09-information-theory",
"file": "phases/01-math-foundations/09-information-theory/outputs/skill-information-theory.md"
},
{
"kind": "skill",
"name": "skill-dimensionality-reduction",
"description": "Choose the right dimensionality reduction technique for a given task based on data size, goal, and downstream use",
"tags": [],
"phase": 1,
"lesson": 10,
"lessonPath": "phases/01-math-foundations/10-dimensionality-reduction",
"file": "phases/01-math-foundations/10-dimensionality-reduction/outputs/skill-dimensionality-reduction.md"
},
{
"kind": "skill",
"name": "skill-svd",
"description": "Apply SVD to real problems including compression, denoising, recommendations, and least-squares solving",
"tags": [],
"phase": 1,
"lesson": 11,
"lessonPath": "phases/01-math-foundations/11-singular-value-decomposition",
"file": "phases/01-math-foundations/11-singular-value-decomposition/outputs/skill-svd.md"
},
{
"kind": "prompt",
"name": "prompt-tensor-debugger",
"description": "Step-by-step debugging prompt for tensor shape errors in deep learning code",
"tags": [],
"phase": 1,
"lesson": 12,
"lessonPath": "phases/01-math-foundations/12-tensor-operations",
"file": "phases/01-math-foundations/12-tensor-operations/outputs/prompt-tensor-debugger.md"
},
{
"kind": "prompt",
"name": "prompt-tensor-shapes",
"description": "Debug tensor shape mismatches and recommend fixes for common deep learning operations",
"tags": [],
"phase": 1,
"lesson": 12,
"lessonPath": "phases/01-math-foundations/12-tensor-operations",
"file": "phases/01-math-foundations/12-tensor-operations/outputs/prompt-tensor-shapes.md"
},
{
"kind": "prompt",
"name": "prompt-numerical-debugger",
"description": "Diagnoses NaN, Inf, and numerical stability issues in neural network training",
"tags": [],
"phase": 1,
"lesson": 13,
"lessonPath": "phases/01-math-foundations/13-numerical-stability",
"file": "phases/01-math-foundations/13-numerical-stability/outputs/prompt-numerical-debugger.md"
},
{
"kind": "prompt",
"name": "prompt-distance-chooser",
"description": "Guides the user through choosing the right distance metric for their specific task",
"tags": [],
"phase": 1,
"lesson": 14,
"lessonPath": "phases/01-math-foundations/14-norms-and-distances",
"file": "phases/01-math-foundations/14-norms-and-distances/outputs/prompt-distance-chooser.md"
},
{
"kind": "skill",
"name": "skill-statistical-testing",
"description": "Choose the right statistical test for comparing ML models and evaluating experiments",
"tags": [
"statistics",
"hypothesis-testing",
"model-comparison"
],
"phase": 1,
"lesson": 15,
"lessonPath": "phases/01-math-foundations/15-statistics-for-ml",
"file": "phases/01-math-foundations/15-statistics-for-ml/outputs/skill-statistical-testing.md"
},
{
"kind": "skill",
"name": "skill-sampling-strategy",
"description": "Choose the right sampling method for generation, estimation, or inference",
"tags": [
"sampling",
"mcmc",
"generation"
],
"phase": 1,
"lesson": 16,
"lessonPath": "phases/01-math-foundations/16-sampling-methods",
"file": "phases/01-math-foundations/16-sampling-methods/outputs/skill-sampling-strategy.md"
},
{
"kind": "prompt",
"name": "prompt-linear-solver",
"description": "Recommend the right algorithm for solving a linear system Ax=b based on matrix properties",
"tags": [],
"phase": 1,
"lesson": 17,
"lessonPath": "phases/01-math-foundations/17-linear-systems",
"file": "phases/01-math-foundations/17-linear-systems/outputs/prompt-linear-solver.md"
},
{
"kind": "skill",
"name": "skill-convexity-checker",
"description": "Determine if an optimization problem is convex and choose the right solver",
"tags": [
"optimization",
"convexity",
"solvers"
],
"phase": 1,
"lesson": 18,
"lessonPath": "phases/01-math-foundations/18-convex-optimization",
"file": "phases/01-math-foundations/18-convex-optimization/outputs/skill-convexity-checker.md"
},
{
"kind": "skill",
"name": "skill-complex-arithmetic",
"description": "Quick reference for complex number operations in ML and signal processing contexts",
"tags": [],
"phase": 1,
"lesson": 19,
"lessonPath": "phases/01-math-foundations/19-complex-numbers",
"file": "phases/01-math-foundations/19-complex-numbers/outputs/skill-complex-arithmetic.md"
},
{
"kind": "prompt",
"name": "prompt-spectral-analyzer",
"description": "Guides analysis of frequency content in signals using Fourier transform techniques",
"tags": [],
"phase": 1,
"lesson": 20,
"lessonPath": "phases/01-math-foundations/20-fourier-transform",
"file": "phases/01-math-foundations/20-fourier-transform/outputs/prompt-spectral-analyzer.md"
},
{
"kind": "skill",
"name": "skill-graph-analysis",
"description": "Analyze graph-structured data and choose the right graph algorithm for ML tasks",
"tags": [],
"phase": 1,
"lesson": 21,
"lessonPath": "phases/01-math-foundations/21-graph-theory",
"file": "phases/01-math-foundations/21-graph-theory/outputs/skill-graph-analysis.md"
},
{
"kind": "prompt",
"name": "prompt-stochastic-process-advisor",
"description": "Identify which stochastic process framework applies to a given problem and recommend implementation",
"tags": [],
"phase": 1,
"lesson": 22,
"lessonPath": "phases/01-math-foundations/22-stochastic-processes",
"file": "phases/01-math-foundations/22-stochastic-processes/outputs/prompt-stochastic-process-advisor.md"
},
{
"kind": "prompt",
"name": "prompt-ml-problem-framer",
"description": "Frame a real-world business problem as a machine learning task",
"tags": [],
"phase": 2,
"lesson": 1,
"lessonPath": "phases/02-ml-fundamentals/01-what-is-machine-learning",
"file": "phases/02-ml-fundamentals/01-what-is-machine-learning/outputs/prompt-ml-problem-framer.md"
},
{
"kind": "skill",
"name": "skill-regression",
"description": "Choose the right regression approach based on data characteristics and problem constraints",
"tags": [
"regression",
"linear-regression",
"polynomial-regression",
"ridge",
"regularization"
],
"phase": 2,
"lesson": 2,
"lessonPath": "phases/02-ml-fundamentals/02-linear-regression",
"file": "phases/02-ml-fundamentals/02-linear-regression/outputs/skill-regression.md"
},
{
"kind": "skill",
"name": "skill-classification-baseline",
"description": "Establish a strong classification baseline before reaching for complex models",
"tags": [
"classification",
"logistic-regression",
"baseline",
"preprocessing"
],
"phase": 2,
"lesson": 3,
"lessonPath": "phases/02-ml-fundamentals/03-logistic-regression",
"file": "phases/02-ml-fundamentals/03-logistic-regression/outputs/skill-classification-baseline.md"
},
{
"kind": "prompt",
"name": "prompt-tree-interpreter",
"description": "Interpret decision tree results and diagnose potential issues",
"tags": [],
"phase": 2,
"lesson": 4,
"lessonPath": "phases/02-ml-fundamentals/04-decision-trees",
"file": "phases/02-ml-fundamentals/04-decision-trees/outputs/prompt-tree-interpreter.md"
},
{
"kind": "skill",
"name": "skill-svm-kernel-chooser",
"description": "Choose the right SVM kernel and tune C and gamma for your problem",
"tags": [
"svm",
"kernel",
"classification",
"hyperparameter-tuning"
],
"phase": 2,
"lesson": 5,
"lessonPath": "phases/02-ml-fundamentals/05-support-vector-machines",
"file": "phases/02-ml-fundamentals/05-support-vector-machines/outputs/skill-svm-kernel-chooser.md"
},
{
"kind": "prompt",
"name": "prompt-distance-metric-advisor",
"description": "Recommend the right distance metric based on data type and problem characteristics",
"tags": [],
"phase": 2,
"lesson": 6,
"lessonPath": "phases/02-ml-fundamentals/06-knn-and-distances",
"file": "phases/02-ml-fundamentals/06-knn-and-distances/outputs/prompt-distance-metric-advisor.md"
},
{
"kind": "skill",
"name": "skill-clustering-guide",
"description": "Choose the right clustering algorithm based on data shape, noise, and constraints",
"tags": [
"clustering",
"k-means",
"dbscan",
"hierarchical",
"gmm",
"unsupervised"
],
"phase": 2,
"lesson": 7,
"lessonPath": "phases/02-ml-fundamentals/07-unsupervised-learning",
"file": "phases/02-ml-fundamentals/07-unsupervised-learning/outputs/skill-clustering-guide.md"
},
{
"kind": "prompt",
"name": "prompt-feature-engineer",
"description": "Systematic prompt for engineering features from raw tabular data",
"tags": [],
"phase": 2,
"lesson": 8,
"lessonPath": "phases/02-ml-fundamentals/08-feature-engineering",
"file": "phases/02-ml-fundamentals/08-feature-engineering/outputs/prompt-feature-engineer.md"
},
{
"kind": "skill",
"name": "skill-evaluation",
"description": "Evaluation strategy checklist for classification and regression models",
"tags": [
"evaluation",
"metrics",
"cross-validation",
"model-selection"
],
"phase": 2,
"lesson": 9,
"lessonPath": "phases/02-ml-fundamentals/09-model-evaluation",
"file": "phases/02-ml-fundamentals/09-model-evaluation/outputs/skill-evaluation.md"
},
{
"kind": "prompt",
"name": "prompt-model-diagnostics",
"description": "Diagnose model performance issues using train/test metrics and learning curves",
"tags": [],
"phase": 2,
"lesson": 10,
"lessonPath": "phases/02-ml-fundamentals/10-bias-variance",
"file": "phases/02-ml-fundamentals/10-bias-variance/outputs/prompt-model-diagnostics.md"
},
{
"kind": "prompt",
"name": "prompt-ensemble-selector",
"description": "Pick the right ensemble method for a given dataset and problem",
"tags": [],
"phase": 2,
"lesson": 11,
"lessonPath": "phases/02-ml-fundamentals/11-ensemble-methods",
"file": "phases/02-ml-fundamentals/11-ensemble-methods/outputs/prompt-ensemble-selector.md"
},
{
"kind": "skill",
"name": "skill-ensemble-builder",
"description": "Choose the right ensemble method and configure it for your problem",
"tags": [
"ensemble",
"bagging",
"boosting",
"random-forest",
"xgboost",
"stacking"
],
"phase": 2,
"lesson": 11,
"lessonPath": "phases/02-ml-fundamentals/11-ensemble-methods",
"file": "phases/02-ml-fundamentals/11-ensemble-methods/outputs/skill-ensemble-builder.md"
},
{
"kind": "prompt",
"name": "prompt-tuning-strategy",
"description": "Recommend a hyperparameter tuning strategy based on model type, data size, and compute budget",
"tags": [],
"phase": 2,
"lesson": 12,
"lessonPath": "phases/02-ml-fundamentals/12-hyperparameter-tuning",
"file": "phases/02-ml-fundamentals/12-hyperparameter-tuning/outputs/prompt-tuning-strategy.md"
},
{
"kind": "prompt",
"name": "prompt-ml-pipeline",
"description": "Build, debug, and deploy reproducible ML pipelines",
"tags": [],
"phase": 2,
"lesson": 13,
"lessonPath": "phases/02-ml-fundamentals/13-ml-pipelines",
"file": "phases/02-ml-fundamentals/13-ml-pipelines/outputs/prompt-ml-pipeline.md"
},
{
"kind": "skill",
"name": "skill-naive-bayes-chooser",
"description": "Choose the right Naive Bayes variant for your classification task",
"tags": [],
"phase": 2,
"lesson": 14,
"lessonPath": "phases/02-ml-fundamentals/14-naive-bayes",
"file": "phases/02-ml-fundamentals/14-naive-bayes/outputs/skill-naive-bayes-chooser.md"
},
{
"kind": "prompt",
"name": "prompt-time-series-advisor",
"description": "Frame time series problems and recommend approaches",
"tags": [],
"phase": 2,
"lesson": 15,
"lessonPath": "phases/02-ml-fundamentals/15-time-series",
"file": "phases/02-ml-fundamentals/15-time-series/outputs/prompt-time-series-advisor.md"
},
{
"kind": "skill",
"name": "skill-anomaly-detector",
"description": "Choose the right anomaly detection approach for your problem",
"tags": [],
"phase": 2,
"lesson": 16,
"lessonPath": "phases/02-ml-fundamentals/16-anomaly-detection",
"file": "phases/02-ml-fundamentals/16-anomaly-detection/outputs/skill-anomaly-detector.md"
},
{
"kind": "skill",
"name": "skill-imbalanced-data",
"description": "Decision checklist for handling imbalanced classification problems",
"tags": [
"imbalanced-data",
"smote",
"class-weights",
"threshold-tuning",
"evaluation"
],
"phase": 2,
"lesson": 17,
"lessonPath": "phases/02-ml-fundamentals/17-imbalanced-data",
"file": "phases/02-ml-fundamentals/17-imbalanced-data/outputs/skill-imbalanced-data.md"
},
{
"kind": "skill",
"name": "skill-feature-selector",
"description": "Quick reference decision tree for choosing the right feature selection method",
"tags": [
"feature-selection",
"mutual-information",
"rfe",
"lasso",
"tree-importance"
],
"phase": 2,
"lesson": 18,
"lessonPath": "phases/02-ml-fundamentals/18-feature-selection",
"file": "phases/02-ml-fundamentals/18-feature-selection/outputs/skill-feature-selector.md"
},
{
"kind": "skill",
"name": "skill-perceptron",
"description": "Understand the perceptron pattern and when to use single-layer vs multi-layer architectures",
"tags": [
"perceptron",
"neural-networks",
"classification",
"deep-learning"
],
"phase": 3,
"lesson": 1,
"lessonPath": "phases/03-deep-learning-core/01-the-perceptron",
"file": "phases/03-deep-learning-core/01-the-perceptron/outputs/skill-perceptron.md"
},
{
"kind": "prompt",
"name": "prompt-network-architect",
"description": "Guides the user through designing neural network architectures by choosing layer counts, neuron counts, and activation functions for a given problem",
"tags": [],
"phase": 3,
"lesson": 2,
"lessonPath": "phases/03-deep-learning-core/02-multi-layer-networks",
"file": "phases/03-deep-learning-core/02-multi-layer-networks/outputs/prompt-network-architect.md"
},
{
"kind": "prompt",
"name": "prompt-gradient-debugger",
"description": "Diagnose and fix gradient problems in neural networks -- vanishing gradients, exploding gradients, and NaN values",
"tags": [],
"phase": 3,
"lesson": 3,
"lessonPath": "phases/03-deep-learning-core/03-backpropagation",
"file": "phases/03-deep-learning-core/03-backpropagation/outputs/prompt-gradient-debugger.md"
},
{
"kind": "prompt",
"name": "prompt-activation-selector",
"description": "A decision prompt for choosing the right activation function for any neural network architecture",
"tags": [],
"phase": 3,
"lesson": 4,
"lessonPath": "phases/03-deep-learning-core/04-activation-functions",
"file": "phases/03-deep-learning-core/04-activation-functions/outputs/prompt-activation-selector.md"
},
{
"kind": "prompt",
"name": "prompt-loss-debugger",
"description": "A diagnostic prompt for debugging loss curves and training failures",
"tags": [],
"phase": 3,
"lesson": 5,
"lessonPath": "phases/03-deep-learning-core/05-loss-functions",
"file": "phases/03-deep-learning-core/05-loss-functions/outputs/prompt-loss-debugger.md"
},
{
"kind": "prompt",
"name": "prompt-loss-function-selector",
"description": "A decision prompt for choosing the right loss function for any ML task",
"tags": [],
"phase": 3,
"lesson": 5,
"lessonPath": "phases/03-deep-learning-core/05-loss-functions",
"file": "phases/03-deep-learning-core/05-loss-functions/outputs/prompt-loss-function-selector.md"
},
{
"kind": "prompt",
"name": "prompt-optimizer-selector",
"description": "A decision prompt for choosing the right optimizer and learning rate for any architecture",
"tags": [],
"phase": 3,
"lesson": 6,
"lessonPath": "phases/03-deep-learning-core/06-optimizers",
"file": "phases/03-deep-learning-core/06-optimizers/outputs/prompt-optimizer-selector.md"
},
{
"kind": "prompt",
"name": "prompt-regularization-advisor",
"description": "A diagnostic prompt for choosing regularization strategies based on overfitting symptoms",
"tags": [],
"phase": 3,
"lesson": 7,
"lessonPath": "phases/03-deep-learning-core/07-regularization",
"file": "phases/03-deep-learning-core/07-regularization/outputs/prompt-regularization-advisor.md"
},
{
"kind": "prompt",
"name": "prompt-init-strategy",
"description": "Diagnose weight initialization problems and recommend the right strategy for any neural network architecture",
"tags": [],
"phase": 3,
"lesson": 8,
"lessonPath": "phases/03-deep-learning-core/08-weight-initialization",
"file": "phases/03-deep-learning-core/08-weight-initialization/outputs/prompt-init-strategy.md"
},
{
"kind": "prompt",
"name": "prompt-lr-schedule-advisor",
"description": "Recommend the right learning rate schedule and hyperparameters for any training setup",
"tags": [],
"phase": 3,
"lesson": 9,
"lessonPath": "phases/03-deep-learning-core/09-learning-rate-schedules",
"file": "phases/03-deep-learning-core/09-learning-rate-schedules/outputs/prompt-lr-schedule-advisor.md"
},
{
"kind": "prompt",
"name": "prompt-framework-architect",
"description": "Design neural network architectures using framework abstractions -- modules, containers, losses, and optimizers",
"tags": [],
"phase": 3,
"lesson": 10,
"lessonPath": "phases/03-deep-learning-core/10-mini-framework",
"file": "phases/03-deep-learning-core/10-mini-framework/outputs/prompt-framework-architect.md"
},
{
"kind": "prompt",
"name": "prompt-pytorch-debugger",
"description": "Diagnose and fix common PyTorch training failures from symptoms",
"tags": [],
"phase": 3,
"lesson": 11,
"lessonPath": "phases/03-deep-learning-core/11-intro-to-pytorch",
"file": "phases/03-deep-learning-core/11-intro-to-pytorch/outputs/prompt-pytorch-debugger.md"
},
{
"kind": "skill",
"name": "skill-pytorch-patterns",
"description": "Reference patterns for PyTorch training, evaluation, and deployment",
"tags": [
"pytorch",
"training",
"deep-learning",
"gpu",
"patterns"
],
"phase": 3,
"lesson": 11,
"lessonPath": "phases/03-deep-learning-core/11-intro-to-pytorch",
"file": "phases/03-deep-learning-core/11-intro-to-pytorch/outputs/skill-pytorch-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-jax-optimizer",
"description": "Choose and configure the right JAX/Optax optimizer for a given training scenario",
"tags": [],
"phase": 3,
"lesson": 12,
"lessonPath": "phases/03-deep-learning-core/12-intro-to-jax",
"file": "phases/03-deep-learning-core/12-intro-to-jax/outputs/prompt-jax-optimizer.md"
},
{
"kind": "skill",
"name": "skill-jax-patterns",
"description": "Functional programming patterns in JAX -- when and how to use grad, jit, vmap, and pmap",
"tags": [
"jax",
"functional-programming",
"autodiff",
"compilation",
"vectorization"
],
"phase": 3,
"lesson": 12,
"lessonPath": "phases/03-deep-learning-core/12-intro-to-jax",
"file": "phases/03-deep-learning-core/12-intro-to-jax/outputs/skill-jax-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-nn-debugger",
"description": "Diagnose neural network training failures from symptoms -- loss curves, gradient stats, and activation patterns",
"tags": [],
"phase": 3,
"lesson": 13,
"lessonPath": "phases/03-deep-learning-core/13-debugging-neural-networks",
"file": "phases/03-deep-learning-core/13-debugging-neural-networks/outputs/prompt-nn-debugger.md"
},
{
"kind": "skill",
"name": "skill-debug-checklist",
"description": "Decision-tree checklist for debugging neural network training failures",
"tags": [
"debugging",
"neural-networks",
"training",
"diagnostics",
"deep-learning"
],
"phase": 3,
"lesson": 13,
"lessonPath": "phases/03-deep-learning-core/13-debugging-neural-networks",
"file": "phases/03-deep-learning-core/13-debugging-neural-networks/outputs/skill-debug-checklist.md"
},
{
"kind": "prompt",
"name": "prompt-vision-preprocessing-audit",
"description": "Turn any model card or dataset card into a checklist of the preprocessing invariants a vision pipeline must honour",
"tags": [],
"phase": 4,
"lesson": 1,
"lessonPath": "phases/04-computer-vision/01-image-fundamentals",
"file": "phases/04-computer-vision/01-image-fundamentals/outputs/prompt-vision-preprocessing-audit.md"
},
{
"kind": "skill",
"name": "skill-image-tensor-inspector",
"description": "Inspect any image-shaped tensor or array and report dtype, layout, range, and whether it looks raw, normalized, or standardized",
"tags": [
"computer-vision",
"debugging",
"preprocessing",
"tensors"
],
"phase": 4,
"lesson": 1,
"lessonPath": "phases/04-computer-vision/01-image-fundamentals",
"file": "phases/04-computer-vision/01-image-fundamentals/outputs/skill-image-tensor-inspector.md"
},
{
"kind": "prompt",
"name": "prompt-cnn-architect",
"description": "Design a stack of Conv2d layers from input size, parameter budget, and target receptive field",
"tags": [],
"phase": 4,
"lesson": 2,
"lessonPath": "phases/04-computer-vision/02-convolutions-from-scratch",
"file": "phases/04-computer-vision/02-convolutions-from-scratch/outputs/prompt-cnn-architect.md"
},
{
"kind": "skill",
"name": "skill-conv-shape-calculator",
"description": "Walk a CNN spec layer by layer and report output shape, receptive field, and parameter count for every block",
"tags": [
"computer-vision",
"cnn",
"architecture",
"debugging"
],
"phase": 4,
"lesson": 2,
"lessonPath": "phases/04-computer-vision/02-convolutions-from-scratch",
"file": "phases/04-computer-vision/02-convolutions-from-scratch/outputs/skill-conv-shape-calculator.md"
},
{
"kind": "prompt",
"name": "prompt-backbone-selector",
"description": "Pick the right vision backbone (LeNet, VGG, ResNet, MobileNet, EfficientNet-Lite, ConvNeXt, ViT) for a given task, dataset size, and compute budget",
"tags": [],
"phase": 4,
"lesson": 3,
"lessonPath": "phases/04-computer-vision/03-cnns-lenet-to-resnet",
"file": "phases/04-computer-vision/03-cnns-lenet-to-resnet/outputs/prompt-backbone-selector.md"
},
{
"kind": "skill",
"name": "skill-residual-block-reviewer",
"description": "Review a PyTorch residual block for skip-connection correctness, BN placement, activation order, and shape alignment",
"tags": [
"computer-vision",
"resnet",
"code-review",
"pytorch"
],
"phase": 4,
"lesson": 3,
"lessonPath": "phases/04-computer-vision/03-cnns-lenet-to-resnet",
"file": "phases/04-computer-vision/03-cnns-lenet-to-resnet/outputs/skill-residual-block-reviewer.md"
},
{
"kind": "prompt",
"name": "prompt-classifier-pipeline-auditor",
"description": "Audit a PyTorch image classification training script for the five invariants that cover most silent bugs",
"tags": [],
"phase": 4,
"lesson": 4,
"lessonPath": "phases/04-computer-vision/04-image-classification",
"file": "phases/04-computer-vision/04-image-classification/outputs/prompt-classifier-pipeline-auditor.md"
},
{
"kind": "skill",
"name": "skill-classification-diagnostics",
"description": "Given a confusion matrix and class names, surface per-class failures and propose the single most impactful fix",
"tags": [
"computer-vision",
"classification",
"evaluation",
"debugging"
],
"phase": 4,
"lesson": 4,
"lessonPath": "phases/04-computer-vision/04-image-classification",
"file": "phases/04-computer-vision/04-image-classification/outputs/skill-classification-diagnostics.md"
},
{
"kind": "prompt",
"name": "prompt-fine-tune-planner",
"description": "Pick feature extraction vs progressive vs end-to-end fine-tuning given dataset size, domain distance, and compute budget",
"tags": [],
"phase": 4,
"lesson": 5,
"lessonPath": "phases/04-computer-vision/05-transfer-learning",
"file": "phases/04-computer-vision/05-transfer-learning/outputs/prompt-fine-tune-planner.md"
},
{
"kind": "skill",
"name": "skill-freeze-inspector",
"description": "Report which parameters are trainable, which BatchNorm layers are in eval mode, and whether the optimizer is actually consuming the trainable parameters",
"tags": [
"computer-vision",
"transfer-learning",
"debugging",
"pytorch"
],
"phase": 4,
"lesson": 5,
"lessonPath": "phases/04-computer-vision/05-transfer-learning",
"file": "phases/04-computer-vision/05-transfer-learning/outputs/skill-freeze-inspector.md"
},
{
"kind": "prompt",
"name": "prompt-detection-metric-reader",
"description": "Turn a precision/recall/AP/mAP row into a one-line diagnosis and the single most useful next experiment",
"tags": [],
"phase": 4,
"lesson": 6,
"lessonPath": "phases/04-computer-vision/06-object-detection-yolo",
"file": "phases/04-computer-vision/06-object-detection-yolo/outputs/prompt-detection-metric-reader.md"
},
{
"kind": "skill",
"name": "skill-anchor-designer",
"description": "Given a dataset of ground-truth boxes, run k-means on (w, h) and return anchor sets per FPN level plus coverage statistics",
"tags": [
"computer-vision",
"detection",
"anchors",
"kmeans"
],
"phase": 4,
"lesson": 6,
"lessonPath": "phases/04-computer-vision/06-object-detection-yolo",
"file": "phases/04-computer-vision/06-object-detection-yolo/outputs/skill-anchor-designer.md"
},
{
"kind": "prompt",
"name": "prompt-segmentation-task-picker",
"description": "Pick semantic vs instance vs panoptic segmentation and name the architecture for a given task",
"tags": [],
"phase": 4,
"lesson": 7,
"lessonPath": "phases/04-computer-vision/07-semantic-segmentation-unet",
"file": "phases/04-computer-vision/07-semantic-segmentation-unet/outputs/prompt-segmentation-task-picker.md"
},
{
"kind": "skill",
"name": "skill-segmentation-mask-inspector",
"description": "Report class distribution, predicted-mask statistics, and the classes most likely to be under-predicted or boundary-blurred",
"tags": [
"computer-vision",
"segmentation",
"debugging",
"evaluation"
],
"phase": 4,
"lesson": 7,
"lessonPath": "phases/04-computer-vision/07-semantic-segmentation-unet",
"file": "phases/04-computer-vision/07-semantic-segmentation-unet/outputs/skill-segmentation-mask-inspector.md"
},
{
"kind": "prompt",
"name": "prompt-instance-vs-semantic-router",
"description": "Ask three questions and pick instance vs semantic vs panoptic segmentation plus the first model",
"tags": [],
"phase": 4,
"lesson": 8,
"lessonPath": "phases/04-computer-vision/08-instance-segmentation-mask-rcnn",
"file": "phases/04-computer-vision/08-instance-segmentation-mask-rcnn/outputs/prompt-instance-vs-semantic-router.md"
},
{
"kind": "skill",
"name": "skill-mask-rcnn-head-swapper",
"description": "Generate the exact code for swapping box and mask heads on a torchvision Mask R-CNN for a custom num_classes",
"tags": [
"computer-vision",
"mask-rcnn",
"fine-tuning",
"torchvision"
],
"phase": 4,
"lesson": 8,
"lessonPath": "phases/04-computer-vision/08-instance-segmentation-mask-rcnn",
"file": "phases/04-computer-vision/08-instance-segmentation-mask-rcnn/outputs/skill-mask-rcnn-head-swapper.md"
},
{
"kind": "prompt",
"name": "prompt-gan-training-triage",
"description": "Read a description of GAN training curves and pick the failure mode plus the single recommended fix",
"tags": [],
"phase": 4,
"lesson": 9,
"lessonPath": "phases/04-computer-vision/09-image-generation-gans",
"file": "phases/04-computer-vision/09-image-generation-gans/outputs/prompt-gan-training-triage.md"
},
{
"kind": "skill",
"name": "skill-dcgan-scaffold",
"description": "Write a complete DCGAN scaffold from z_dim, image_size, and num_channels, including training loop and sample saver",
"tags": [
"computer-vision",
"gan",
"dcgan",
"scaffolding"
],
"phase": 4,
"lesson": 9,
"lessonPath": "phases/04-computer-vision/09-image-generation-gans",
"file": "phases/04-computer-vision/09-image-generation-gans/outputs/skill-dcgan-scaffold.md"
},
{
"kind": "prompt",
"name": "prompt-diffusion-sampler-picker",
"description": "Pick DDPM, DDIM, DPM-Solver++, or Euler ancestral based on quality target, latency budget, and conditioning type",
"tags": [],
"phase": 4,
"lesson": 10,
"lessonPath": "phases/04-computer-vision/10-image-generation-diffusion",
"file": "phases/04-computer-vision/10-image-generation-diffusion/outputs/prompt-diffusion-sampler-picker.md"
},
{
"kind": "skill",
"name": "skill-noise-schedule-designer",
"description": "Produce a linear, cosine, or sigmoid beta schedule given T and target corruption level, plus SNR plot",
"tags": [
"computer-vision",
"diffusion",
"noise-schedule",
"training"
],
"phase": 4,
"lesson": 10,
"lessonPath": "phases/04-computer-vision/10-image-generation-diffusion",
"file": "phases/04-computer-vision/10-image-generation-diffusion/outputs/skill-noise-schedule-designer.md"
},
{
"kind": "prompt",
"name": "prompt-sd-pipeline-planner",
"description": "Pick SD 1.5 / SDXL / SD3 / FLUX plus scheduler and precision given a latency budget, fidelity target, and licensing constraint",
"tags": [],
"phase": 4,
"lesson": 11,
"lessonPath": "phases/04-computer-vision/11-stable-diffusion",
"file": "phases/04-computer-vision/11-stable-diffusion/outputs/prompt-sd-pipeline-planner.md"
},
{
"kind": "skill",
"name": "skill-lora-training-setup",
"description": "Write a full LoRA training config for a custom dataset, including captions, rank, batch size, and learning rate",
"tags": [
"computer-vision",
"stable-diffusion",
"lora",
"fine-tuning"
],
"phase": 4,
"lesson": 11,
"lessonPath": "phases/04-computer-vision/11-stable-diffusion",
"file": "phases/04-computer-vision/11-stable-diffusion/outputs/skill-lora-training-setup.md"
},
{
"kind": "prompt",
"name": "prompt-video-architecture-picker",
"description": "Pick 2D+pool / I3D / (2+1)D / spatio-temporal transformer based on appearance-vs-motion, dataset size, and compute budget",
"tags": [],
"phase": 4,
"lesson": 12,
"lessonPath": "phases/04-computer-vision/12-video-understanding",
"file": "phases/04-computer-vision/12-video-understanding/outputs/prompt-video-architecture-picker.md"
},
{
"kind": "skill",
"name": "skill-frame-sampler-auditor",
"description": "Audit a video pipeline's frame sampler for off-by-one, short-clip handling, and crop consistency",
"tags": [
"computer-vision",
"video",
"sampling",
"debugging"
],
"phase": 4,
"lesson": 12,
"lessonPath": "phases/04-computer-vision/12-video-understanding",
"file": "phases/04-computer-vision/12-video-understanding/outputs/skill-frame-sampler-auditor.md"
},
{
"kind": "prompt",
"name": "prompt-3d-task-router",
"description": "Route to the right 3D representation (point cloud, mesh, voxel, NeRF, Gaussian splat) based on task and input",
"tags": [],
"phase": 4,
"lesson": 13,
"lessonPath": "phases/04-computer-vision/13-3d-vision-nerf",
"file": "phases/04-computer-vision/13-3d-vision-nerf/outputs/prompt-3d-task-router.md"
},
{
"kind": "skill",
"name": "skill-point-cloud-loader",
"description": "Write a PyTorch Dataset for .ply / .pcd / .xyz files with correct normalisation, centring, and point sampling",
"tags": [
"3d-vision",
"point-cloud",
"data-loading",
"pytorch"
],
"phase": 4,
"lesson": 13,
"lessonPath": "phases/04-computer-vision/13-3d-vision-nerf",
"file": "phases/04-computer-vision/13-3d-vision-nerf/outputs/skill-point-cloud-loader.md"
},
{
"kind": "prompt",
"name": "prompt-vit-vs-cnn-picker",
"description": "Pick between ViT, ConvNeXt, or Swin based on dataset size, compute, and inference stack",
"tags": [],
"phase": 4,
"lesson": 14,
"lessonPath": "phases/04-computer-vision/14-vision-transformers",
"file": "phases/04-computer-vision/14-vision-transformers/outputs/prompt-vit-vs-cnn-picker.md"
},
{
"kind": "skill",
"name": "skill-vit-patch-and-pos-embed-inspector",
"description": "Verify a ViT's patch embedding and positional embedding shapes match the model's expected sequence length",
"tags": [
"vision-transformer",
"debugging",
"pytorch"
],
"phase": 4,
"lesson": 14,
"lessonPath": "phases/04-computer-vision/14-vision-transformers",
"file": "phases/04-computer-vision/14-vision-transformers/outputs/skill-vit-patch-and-pos-embed-inspector.md"
},
{
"kind": "prompt",
"name": "prompt-edge-deployment-planner",
"description": "Pick backbone, quantisation strategy, and runtime given target device and latency SLA",
"tags": [],
"phase": 4,
"lesson": 15,
"lessonPath": "phases/04-computer-vision/15-real-time-edge",
"file": "phases/04-computer-vision/15-real-time-edge/outputs/prompt-edge-deployment-planner.md"
},
{
"kind": "skill",
"name": "skill-latency-profiler",
"description": "Write a complete latency-benchmarking script with warmup, synchronisation, percentiles, and memory tracking",
"tags": [
"edge",
"deployment",
"profiling",
"benchmarking"
],
"phase": 4,
"lesson": 15,
"lessonPath": "phases/04-computer-vision/15-real-time-edge",
"file": "phases/04-computer-vision/15-real-time-edge/outputs/skill-latency-profiler.md"
},
{
"kind": "prompt",
"name": "prompt-vision-service-shape-reviewer",
"description": "Review a vision service's code for contract/response shape violations and name the first breaking bug",
"tags": [],
"phase": 4,
"lesson": 16,
"lessonPath": "phases/04-computer-vision/16-vision-pipeline-capstone",
"file": "phases/04-computer-vision/16-vision-pipeline-capstone/outputs/prompt-vision-service-shape-reviewer.md"
},
{
"kind": "skill",
"name": "skill-pipeline-budget-planner",
"description": "Given target latency and throughput, assign a time budget to every pipeline stage and flag which stage will miss its budget first",
"tags": [
"vision",
"pipeline",
"performance",
"deployment"
],
"phase": 4,
"lesson": 16,
"lessonPath": "phases/04-computer-vision/16-vision-pipeline-capstone",
"file": "phases/04-computer-vision/16-vision-pipeline-capstone/outputs/skill-pipeline-budget-planner.md"
},
{
"kind": "prompt",
"name": "prompt-ssl-pretraining-picker",
"description": "Pick SimCLR / MAE / DINOv2 given dataset size, compute, and downstream task",
"tags": [],
"phase": 4,
"lesson": 17,
"lessonPath": "phases/04-computer-vision/17-self-supervised-vision",
"file": "phases/04-computer-vision/17-self-supervised-vision/outputs/prompt-ssl-pretraining-picker.md"
},
{
"kind": "skill",
"name": "skill-linear-probe-runner",
"description": "Write the complete linear-probe evaluation for any frozen encoder and labelled dataset",
"tags": [
"self-supervised",
"evaluation",
"linear-probe",
"pytorch"
],
"phase": 4,
"lesson": 17,
"lessonPath": "phases/04-computer-vision/17-self-supervised-vision",
"file": "phases/04-computer-vision/17-self-supervised-vision/outputs/skill-linear-probe-runner.md"
},
{
"kind": "prompt",
"name": "prompt-zero-shot-class-picker",
"description": "Design prompt templates for zero-shot CLIP given a list of classes and a domain",
"tags": [],
"phase": 4,
"lesson": 18,
"lessonPath": "phases/04-computer-vision/18-open-vocab-clip",
"file": "phases/04-computer-vision/18-open-vocab-clip/outputs/prompt-zero-shot-class-picker.md"
},
{
"kind": "skill",
"name": "skill-image-text-retriever",
"description": "Build an image embedding index with any CLIP checkpoint; support query-by-text and query-by-image",
"tags": [
"clip",
"retrieval",
"faiss",
"zero-shot"
],
"phase": 4,
"lesson": 18,
"lessonPath": "phases/04-computer-vision/18-open-vocab-clip",
"file": "phases/04-computer-vision/18-open-vocab-clip/outputs/skill-image-text-retriever.md"
},
{
"kind": "prompt",
"name": "prompt-ocr-stack-picker",
"description": "Pick Tesseract / PaddleOCR / Donut / VLM-OCR given document type, language, and structure",
"tags": [],
"phase": 4,
"lesson": 19,
"lessonPath": "phases/04-computer-vision/19-ocr-document-understanding",
"file": "phases/04-computer-vision/19-ocr-document-understanding/outputs/prompt-ocr-stack-picker.md"
},
{
"kind": "skill",
"name": "skill-ctc-decoder",
"description": "Write greedy and beam-search CTC decoders from scratch, including length normalisation",
"tags": [
"ocr",
"ctc",
"decoding",
"sequence-models"
],
"phase": 4,
"lesson": 19,
"lessonPath": "phases/04-computer-vision/19-ocr-document-understanding",
"file": "phases/04-computer-vision/19-ocr-document-understanding/outputs/skill-ctc-decoder.md"
},
{
"kind": "prompt",
"name": "prompt-retrieval-loss-picker",
"description": "Pick triplet / InfoNCE / ProxyNCA for a given retrieval problem",
"tags": [],
"phase": 4,
"lesson": 20,
"lessonPath": "phases/04-computer-vision/20-image-retrieval-metric",
"file": "phases/04-computer-vision/20-image-retrieval-metric/outputs/prompt-retrieval-loss-picker.md"
},
{
"kind": "skill",
"name": "skill-recall-at-k-runner",
"description": "Write a clean evaluation harness for recall@K with train/val/gallery splits and proper data contract",
"tags": [
"retrieval",
"evaluation",
"recall",
"faiss"
],
"phase": 4,
"lesson": 20,
"lessonPath": "phases/04-computer-vision/20-image-retrieval-metric",
"file": "phases/04-computer-vision/20-image-retrieval-metric/outputs/skill-recall-at-k-runner.md"
},
{
"kind": "prompt",
"name": "prompt-pose-stack-picker",
"description": "Pick MediaPipe / YOLOv8-pose / HRNet / ViTPose given latency, crowd size, and 2D vs 3D need",
"tags": [],
"phase": 4,
"lesson": 21,
"lessonPath": "phases/04-computer-vision/21-keypoint-pose",
"file": "phases/04-computer-vision/21-keypoint-pose/outputs/prompt-pose-stack-picker.md"
},
{
"kind": "skill",
"name": "skill-heatmap-to-coords",
"description": "Write the sub-pixel heatmap-to-coordinate routine used by every production pose model",
"tags": [
"keypoint",
"pose",
"subpixel",
"inference"
],
"phase": 4,
"lesson": 21,
"lessonPath": "phases/04-computer-vision/21-keypoint-pose",
"file": "phases/04-computer-vision/21-keypoint-pose/outputs/skill-heatmap-to-coords.md"
},
{
"kind": "prompt",
"name": "prompt-3dgs-capture-planner",
"description": "Plan a photo capture session for 3DGS reconstruction given scene type and hardware",
"tags": [],
"phase": 4,
"lesson": 22,
"lessonPath": "phases/04-computer-vision/22-3d-gaussian-splatting",
"file": "phases/04-computer-vision/22-3d-gaussian-splatting/outputs/prompt-3dgs-capture-planner.md"
},
{
"kind": "skill",
"name": "skill-3dgs-export-router",
"description": "Pick the right 3DGS export format (.ply / .splat / glTF KHR_gaussian_splatting / USD) given the downstream viewer or engine",
"tags": [
"3d-gaussian-splatting",
"export",
"glTF",
"OpenUSD",
"pipeline"
],
"phase": 4,
"lesson": 22,
"lessonPath": "phases/04-computer-vision/22-3d-gaussian-splatting",
"file": "phases/04-computer-vision/22-3d-gaussian-splatting/outputs/skill-3dgs-export-router.md"
},
{
"kind": "prompt",
"name": "prompt-dit-model-picker",
"description": "Pick between SD3, SD3.5, FLUX.1-dev, FLUX.1-schnell, Z-Image, SD4 Turbo given quality, latency, and license",
"tags": [],
"phase": 4,
"lesson": 23,
"lessonPath": "phases/04-computer-vision/23-diffusion-transformers-rectified-flow",
"file": "phases/04-computer-vision/23-diffusion-transformers-rectified-flow/outputs/prompt-dit-model-picker.md"
},
{
"kind": "skill",
"name": "skill-rectified-flow-trainer",
"description": "Write a complete rectified-flow training loop with AdaLN DiT and Euler sampling",
"tags": [
"diffusion",
"rectified-flow",
"DiT",
"training"
],
"phase": 4,
"lesson": 23,
"lessonPath": "phases/04-computer-vision/23-diffusion-transformers-rectified-flow",
"file": "phases/04-computer-vision/23-diffusion-transformers-rectified-flow/outputs/skill-rectified-flow-trainer.md"
},
{
"kind": "prompt",
"name": "prompt-open-vocab-stack-picker",
"description": "Pick SAM 3 / Grounded SAM 2 / YOLO-World / SAM-MI based on latency, concept complexity, and licensing",
"tags": [],
"phase": 4,
"lesson": 24,
"lessonPath": "phases/04-computer-vision/24-sam3-open-vocab-segmentation",
"file": "phases/04-computer-vision/24-sam3-open-vocab-segmentation/outputs/prompt-open-vocab-stack-picker.md"
},
{
"kind": "skill",
"name": "skill-concept-prompt-designer",
"description": "Turn user utterances into well-formed SAM 3 concept prompts with splitting, disambiguation, and fallbacks",
"tags": [
"sam3",
"open-vocab",
"prompt-engineering",
"segmentation"
],
"phase": 4,
"lesson": 24,
"lessonPath": "phases/04-computer-vision/24-sam3-open-vocab-segmentation",
"file": "phases/04-computer-vision/24-sam3-open-vocab-segmentation/outputs/skill-concept-prompt-designer.md"
},
{
"kind": "prompt",
"name": "prompt-vlm-selector",
"description": "Pick Qwen3-VL / InternVL3.5 / LLaVA-Next / API given accuracy, latency, context length, and budget",
"tags": [],
"phase": 4,
"lesson": 25,
"lessonPath": "phases/04-computer-vision/25-vision-language-models",
"file": "phases/04-computer-vision/25-vision-language-models/outputs/prompt-vlm-selector.md"
},
{
"kind": "skill",
"name": "skill-cmer-monitor",
"description": "Instrument a production VLM endpoint with Cross-Modal Error Rate monitoring, dashboards, and alerts",
"tags": [
"vlm",
"production",
"monitoring",
"hallucination"
],
"phase": 4,
"lesson": 25,
"lessonPath": "phases/04-computer-vision/25-vision-language-models",
"file": "phases/04-computer-vision/25-vision-language-models/outputs/skill-cmer-monitor.md"
},
{
"kind": "prompt",
"name": "prompt-depth-model-picker",
"description": "Pick Depth Anything V3 / Marigold / UniDepth / MiDaS given latency, metric-vs-relative need, and scene type",
"tags": [],
"phase": 4,
"lesson": 26,
"lessonPath": "phases/04-computer-vision/26-monocular-depth",
"file": "phases/04-computer-vision/26-monocular-depth/outputs/prompt-depth-model-picker.md"
},
{
"kind": "skill",
"name": "skill-depth-to-pointcloud",
"description": "Build point clouds from depth maps with correct intrinsics handling and export to .ply",
"tags": [
"depth",
"point-cloud",
"3d",
"intrinsics"
],
"phase": 4,
"lesson": 26,
"lessonPath": "phases/04-computer-vision/26-monocular-depth",
"file": "phases/04-computer-vision/26-monocular-depth/outputs/skill-depth-to-pointcloud.md"
},
{
"kind": "prompt",
"name": "prompt-tracker-picker",
"description": "Pick SORT / ByteTrack / BoT-SORT / SAM 2 / SAM 3.1 given scene type, occlusion patterns, and latency budget",
"tags": [],
"phase": 4,
"lesson": 27,
"lessonPath": "phases/04-computer-vision/27-multi-object-tracking",
"file": "phases/04-computer-vision/27-multi-object-tracking/outputs/prompt-tracker-picker.md"
},
{
"kind": "skill",
"name": "skill-mot-evaluator",
"description": "Write a complete evaluation harness for MOTA / IDF1 / HOTA against ground-truth tracks",
"tags": [
"mot",
"evaluation",
"tracking",
"metrics"
],
"phase": 4,
"lesson": 27,
"lessonPath": "phases/04-computer-vision/27-multi-object-tracking",
"file": "phases/04-computer-vision/27-multi-object-tracking/outputs/skill-mot-evaluator.md"
},
{
"kind": "prompt",
"name": "prompt-video-model-picker",
"description": "Pick Sora 2 / Runway Gen-5 / Wan-Video / HunyuanVideo / Cosmos for a given task, license, and latency target",
"tags": [],
"phase": 4,
"lesson": 28,
"lessonPath": "phases/04-computer-vision/28-world-models-video-diffusion",
"file": "phases/04-computer-vision/28-world-models-video-diffusion/outputs/prompt-video-model-picker.md"
},
{
"kind": "skill",
"name": "skill-physical-plausibility-checks",
"description": "Automated checks for object permanence, gravity, and continuity on any generated video before shipping",
"tags": [
"video-generation",
"quality",
"physics",
"evaluation"
],
"phase": 4,
"lesson": 28,
"lessonPath": "phases/04-computer-vision/28-world-models-video-diffusion",
"file": "phases/04-computer-vision/28-world-models-video-diffusion/outputs/skill-physical-plausibility-checks.md"
},
{
"kind": "prompt",
"name": "preprocessing-advisor",
"description": "Recommends a tokenization, stemming, and lemmatization setup for an NLP task.",
"tags": [],
"phase": 5,
"lesson": 1,
"lessonPath": "phases/05-nlp-foundations-to-advanced/01-text-processing",
"file": "phases/05-nlp-foundations-to-advanced/01-text-processing/outputs/prompt-preprocessing-advisor.md"
},
{
"kind": "prompt",
"name": "vectorization-picker",
"description": "Given a text-classification task, recommend BoW, TF-IDF, embeddings, or a hybrid.",
"tags": [],
"phase": 5,
"lesson": 2,
"lessonPath": "phases/05-nlp-foundations-to-advanced/02-bag-of-words-tfidf",
"file": "phases/05-nlp-foundations-to-advanced/02-bag-of-words-tfidf/outputs/prompt-vectorization-picker.md"
},
{
"kind": "skill",
"name": "embedding-probe",
"description": "Inspect a word2vec model. Run analogies, find neighbors, diagnose quality.",
"tags": [
"nlp",
"embeddings",
"debugging"
],
"phase": 5,
"lesson": 3,
"lessonPath": "phases/05-nlp-foundations-to-advanced/03-word-embeddings-word2vec",
"file": "phases/05-nlp-foundations-to-advanced/03-word-embeddings-word2vec/outputs/skill-embedding-probe.md"
},
{
"kind": "skill",
"name": "skill-embeddings-picker",
"description": "Pick a tokenization approach for a new language model or text pipeline.",
"tags": [
"nlp",
"tokenization",
"embeddings"
],
"phase": 5,
"lesson": 4,
"lessonPath": "phases/05-nlp-foundations-to-advanced/04-glove-fasttext-subword",
"file": "phases/05-nlp-foundations-to-advanced/04-glove-fasttext-subword/outputs/skill-embeddings-picker.md"
},
{
"kind": "prompt",
"name": "sentiment-baseline",
"description": "Design a sentiment analysis baseline for a new dataset.",
"tags": [],
"phase": 5,
"lesson": 5,
"lessonPath": "phases/05-nlp-foundations-to-advanced/05-sentiment-analysis",
"file": "phases/05-nlp-foundations-to-advanced/05-sentiment-analysis/outputs/prompt-sentiment-baseline.md"
},
{
"kind": "skill",
"name": "ner-picker",
"description": "Pick the right NER approach for a given extraction task.",
"tags": [
"nlp",
"ner",
"extraction"
],
"phase": 5,
"lesson": 6,
"lessonPath": "phases/05-nlp-foundations-to-advanced/06-named-entity-recognition",
"file": "phases/05-nlp-foundations-to-advanced/06-named-entity-recognition/outputs/skill-ner-picker.md"
},
{
"kind": "skill",
"name": "grammar-pipeline",
"description": "Design a classical POS + dependency pipeline for a downstream NLP task.",
"tags": [
"nlp",
"pos",
"parsing"
],
"phase": 5,
"lesson": 7,
"lessonPath": "phases/05-nlp-foundations-to-advanced/07-pos-tagging-parsing",
"file": "phases/05-nlp-foundations-to-advanced/07-pos-tagging-parsing/outputs/skill-grammar-pipeline.md"
},
{
"kind": "prompt",
"name": "text-encoder-picker",
"description": "Pick a text encoder architecture for a given constraint set.",
"tags": [],
"phase": 5,
"lesson": 8,
"lessonPath": "phases/05-nlp-foundations-to-advanced/08-cnns-rnns-for-text",
"file": "phases/05-nlp-foundations-to-advanced/08-cnns-rnns-for-text/outputs/prompt-text-encoder-picker.md"
},
{
"kind": "prompt",
"name": "seq2seq-design",
"description": "Design a sequence-to-sequence pipeline for a given task.",
"tags": [],
"phase": 5,
"lesson": 9,
"lessonPath": "phases/05-nlp-foundations-to-advanced/09-sequence-to-sequence",
"file": "phases/05-nlp-foundations-to-advanced/09-sequence-to-sequence/outputs/prompt-seq2seq-design.md"
},
{
"kind": "prompt",
"name": "attention-shapes",
"description": "Debug shape bugs in attention implementations.",
"tags": [],
"phase": 5,
"lesson": 10,
"lessonPath": "phases/05-nlp-foundations-to-advanced/10-attention-mechanism",
"file": "phases/05-nlp-foundations-to-advanced/10-attention-mechanism/outputs/prompt-attention-shapes.md"
},
{
"kind": "skill",
"name": "mt-evaluator",
"description": "Evaluate a machine translation output for shipping.",
"tags": [
"nlp",
"translation",
"evaluation"
],
"phase": 5,
"lesson": 11,
"lessonPath": "phases/05-nlp-foundations-to-advanced/11-machine-translation",
"file": "phases/05-nlp-foundations-to-advanced/11-machine-translation/outputs/skill-mt-evaluator.md"
},
{
"kind": "skill",
"name": "summary-picker",
"description": "Pick extractive or abstractive, name the library, add a factuality check.",
"tags": [
"nlp",
"summarization"
],
"phase": 5,
"lesson": 12,
"lessonPath": "phases/05-nlp-foundations-to-advanced/12-text-summarization",
"file": "phases/05-nlp-foundations-to-advanced/12-text-summarization/outputs/skill-summary-picker.md"
},
{
"kind": "skill",
"name": "qa-architect",
"description": "Choose QA architecture, retrieval strategy, and evaluation plan.",
"tags": [
"nlp",
"qa",
"rag"
],
"phase": 5,
"lesson": 13,
"lessonPath": "phases/05-nlp-foundations-to-advanced/13-question-answering",
"file": "phases/05-nlp-foundations-to-advanced/13-question-answering/outputs/skill-qa-architect.md"
},
{
"kind": "skill",
"name": "retrieval-picker",
"description": "Pick a retrieval stack for a given corpus and query pattern.",
"tags": [
"nlp",
"retrieval",
"rag",
"search"
],
"phase": 5,
"lesson": 14,
"lessonPath": "phases/05-nlp-foundations-to-advanced/14-information-retrieval-search",
"file": "phases/05-nlp-foundations-to-advanced/14-information-retrieval-search/outputs/skill-retrieval-picker.md"
},
{
"kind": "skill",
"name": "topic-picker",
"description": "Pick LDA or BERTopic for a corpus. Specify library, knobs, evaluation.",
"tags": [
"nlp",
"topic-modeling"
],
"phase": 5,
"lesson": 15,
"lessonPath": "phases/05-nlp-foundations-to-advanced/15-topic-modeling",
"file": "phases/05-nlp-foundations-to-advanced/15-topic-modeling/outputs/skill-topic-picker.md"
},
{
"kind": "prompt",
"name": "lm-baseline",
"description": "Build a reproducible n-gram language model baseline before training a neural LM.",
"tags": [],
"phase": 5,
"lesson": 16,
"lessonPath": "phases/05-nlp-foundations-to-advanced/16-text-generation-pre-transformer",
"file": "phases/05-nlp-foundations-to-advanced/16-text-generation-pre-transformer/outputs/prompt-lm-baseline.md"
},
{
"kind": "skill",
"name": "chatbot-architect",
"description": "Design a chatbot stack for a given use case.",
"tags": [
"nlp",
"agents",
"chatbot"
],
"phase": 5,
"lesson": 17,
"lessonPath": "phases/05-nlp-foundations-to-advanced/17-chatbots-rule-to-neural",
"file": "phases/05-nlp-foundations-to-advanced/17-chatbots-rule-to-neural/outputs/skill-chatbot-architect.md"
},
{
"kind": "skill",
"name": "multilingual-picker",
"description": "Pick source language, target model, and evaluation plan for a multilingual NLP task.",
"tags": [
"nlp",
"multilingual",
"cross-lingual"
],
"phase": 5,
"lesson": 18,
"lessonPath": "phases/05-nlp-foundations-to-advanced/18-multilingual-nlp",
"file": "phases/05-nlp-foundations-to-advanced/18-multilingual-nlp/outputs/skill-multilingual-picker.md"
},
{
"kind": "skill",
"name": "skill-bpe-vs-wordpiece",
"description": "Pick tokenizer algorithm, vocab size, library for a given corpus and deployment target.",
"tags": [
"nlp",
"tokenization"
],
"phase": 5,
"lesson": 19,
"lessonPath": "phases/05-nlp-foundations-to-advanced/19-subword-tokenization",
"file": "phases/05-nlp-foundations-to-advanced/19-subword-tokenization/outputs/skill-bpe-vs-wordpiece.md"
},
{
"kind": "skill",
"name": "structured-output-picker",
"description": "Choose a structured output approach, schema design, and validation plan.",
"tags": [
"nlp",
"llm",
"structured-output"
],
"phase": 5,
"lesson": 20,
"lessonPath": "phases/05-nlp-foundations-to-advanced/20-structured-outputs-constrained-decoding",
"file": "phases/05-nlp-foundations-to-advanced/20-structured-outputs-constrained-decoding/outputs/skill-structured-output-picker.md"
},
{
"kind": "skill",
"name": "nli-picker",
"description": "Pick an NLI model, label template, and evaluation setup for a classification / faithfulness / zero-shot task.",
"tags": [
"nlp",
"nli",
"zero-shot"
],
"phase": 5,
"lesson": 21,
"lessonPath": "phases/05-nlp-foundations-to-advanced/21-nli-textual-entailment",
"file": "phases/05-nlp-foundations-to-advanced/21-nli-textual-entailment/outputs/skill-nli-picker.md"
},
{
"kind": "skill",
"name": "embedding-picker",
"description": "Pick embedding model, dimension, and retrieval mode for a given corpus and deployment.",
"tags": [
"nlp",
"embeddings",
"retrieval"
],
"phase": 5,
"lesson": 22,
"lessonPath": "phases/05-nlp-foundations-to-advanced/22-embedding-models-deep-dive",
"file": "phases/05-nlp-foundations-to-advanced/22-embedding-models-deep-dive/outputs/skill-embedding-picker.md"
},
{
"kind": "skill",
"name": "chunker",
"description": "Pick a chunking strategy, size, and overlap for a given corpus and query distribution.",
"tags": [
"nlp",
"rag",
"chunking"
],
"phase": 5,
"lesson": 23,
"lessonPath": "phases/05-nlp-foundations-to-advanced/23-chunking-strategies-rag",
"file": "phases/05-nlp-foundations-to-advanced/23-chunking-strategies-rag/outputs/skill-chunker.md"
},
{
"kind": "skill",
"name": "coref-picker",
"description": "Pick a coreference approach, evaluation plan, and integration strategy.",
"tags": [
"nlp",
"coref",
"information-extraction"
],
"phase": 5,
"lesson": 24,
"lessonPath": "phases/05-nlp-foundations-to-advanced/24-coreference-resolution",
"file": "phases/05-nlp-foundations-to-advanced/24-coreference-resolution/outputs/skill-coref-picker.md"
},
{
"kind": "skill",
"name": "entity-linker",
"description": "Design an entity linking pipeline — KB, candidate generator, disambiguator, evaluation.",
"tags": [
"nlp",
"entity-linking",
"knowledge-graph"
],
"phase": 5,
"lesson": 25,
"lessonPath": "phases/05-nlp-foundations-to-advanced/25-entity-linking",
"file": "phases/05-nlp-foundations-to-advanced/25-entity-linking/outputs/skill-entity-linker.md"
},
{
"kind": "skill",
"name": "re-designer",
"description": "Design a relation extraction pipeline with provenance and canonicalization.",
"tags": [
"nlp",
"relation-extraction",
"knowledge-graph"
],
"phase": 5,
"lesson": 26,
"lessonPath": "phases/05-nlp-foundations-to-advanced/26-relation-extraction-kg",
"file": "phases/05-nlp-foundations-to-advanced/26-relation-extraction-kg/outputs/skill-re-designer.md"
},
{
"kind": "skill",
"name": "eval-architect",
"description": "Design an LLM evaluation plan with calibrated judge and CI gates.",
"tags": [
"nlp",
"evaluation",
"rag"
],
"phase": 5,
"lesson": 27,
"lessonPath": "phases/05-nlp-foundations-to-advanced/27-llm-evaluation-frameworks",
"file": "phases/05-nlp-foundations-to-advanced/27-llm-evaluation-frameworks/outputs/skill-eval-architect.md"
},
{
"kind": "skill",
"name": "long-context-eval",
"description": "Design a long-context evaluation battery for a given model and use case.",
"tags": [
"nlp",
"long-context",
"evaluation"
],
"phase": 5,
"lesson": 28,
"lessonPath": "phases/05-nlp-foundations-to-advanced/28-long-context-evaluation",
"file": "phases/05-nlp-foundations-to-advanced/28-long-context-evaluation/outputs/skill-long-context-eval.md"
},
{
"kind": "skill",
"name": "dst-designer",
"description": "Design a dialogue state tracker — schema, extractor, update policy, evaluation.",
"tags": [
"nlp",
"dialogue",
"task-oriented"
],
"phase": 5,
"lesson": 29,
"lessonPath": "phases/05-nlp-foundations-to-advanced/29-dialogue-state-tracking",
"file": "phases/05-nlp-foundations-to-advanced/29-dialogue-state-tracking/outputs/skill-dst-designer.md"
},
{
"kind": "skill",
"name": "audio-loader",
"description": "Validate a raw audio file against a target model's expectations and resample it safely.",
"tags": [
"audio",
"speech",
"preprocessing"
],
"phase": 6,
"lesson": 1,
"lessonPath": "phases/06-speech-and-audio/01-audio-fundamentals",
"file": "phases/06-speech-and-audio/01-audio-fundamentals/outputs/skill-audio-loader.md"
},
{
"kind": "skill",
"name": "feature-extractor",
"description": "Pick feature type, mel count, frame/hop, and normalization to match a downstream audio model.",
"tags": [
"audio",
"features",
"spectrogram",
"mel"
],
"phase": 6,
"lesson": 2,
"lessonPath": "phases/06-speech-and-audio/02-spectrograms-mel-features",
"file": "phases/06-speech-and-audio/02-spectrograms-mel-features/outputs/skill-feature-extractor.md"
},
{
"kind": "skill",
"name": "classifier-designer",
"description": "Pick architecture, augmentation, class-balance strategy, and eval metric for an audio classification task.",
"tags": [
"audio",
"classification",
"beats",
"ast"
],
"phase": 6,
"lesson": 3,
"lessonPath": "phases/06-speech-and-audio/03-audio-classification",
"file": "phases/06-speech-and-audio/03-audio-classification/outputs/skill-classifier-designer.md"
},
{
"kind": "skill",
"name": "asr-picker",
"description": "Pick ASR model, decoding strategy, chunking, and LM fusion for a given deployment target.",
"tags": [
"audio",
"asr",
"speech-recognition"
],
"phase": 6,
"lesson": 4,
"lessonPath": "phases/06-speech-and-audio/04-speech-recognition-asr",
"file": "phases/06-speech-and-audio/04-speech-recognition-asr/outputs/skill-asr-picker.md"
},
{
"kind": "skill",
"name": "whisper-tuner",
"description": "Design a Whisper fine-tune or inference pipeline for a given language, domain, and latency budget.",
"tags": [
"audio",
"whisper",
"asr",
"fine-tuning",
"lora"
],
"phase": 6,
"lesson": 5,
"lessonPath": "phases/06-speech-and-audio/05-whisper-architecture-finetuning",
"file": "phases/06-speech-and-audio/05-whisper-architecture-finetuning/outputs/skill-whisper-tuner.md"
},
{
"kind": "skill",
"name": "speaker-verifier",
"description": "Design a speaker verification or diarization pipeline with model choice, enrollment protocol, and threshold tuning.",
"tags": [
"audio",
"speaker",
"verification",
"diarization"
],
"phase": 6,
"lesson": 6,
"lessonPath": "phases/06-speech-and-audio/06-speaker-recognition-verification",
"file": "phases/06-speech-and-audio/06-speaker-recognition-verification/outputs/skill-speaker-verifier.md"
},
{
"kind": "skill",
"name": "tts-designer",
"description": "Pick TTS model, voice, text-normalization scope, and evaluation plan for a given language, style, and latency target.",
"tags": [
"audio",
"tts",
"speech-synthesis"
],
"phase": 6,
"lesson": 7,
"lessonPath": "phases/06-speech-and-audio/07-text-to-speech",
"file": "phases/06-speech-and-audio/07-text-to-speech/outputs/skill-tts-designer.md"
},
{
"kind": "skill",
"name": "voice-cloner",
"description": "Pick cloning approach (zero-shot / conversion / adaptation), consent artifact, watermark, and safety filters for a voice-cloning deployment.",
"tags": [
"voice-cloning",
"voice-conversion",
"watermark",
"consent",
"safety"
],
"phase": 6,
"lesson": 8,
"lessonPath": "phases/06-speech-and-audio/08-voice-cloning-conversion",
"file": "phases/06-speech-and-audio/08-voice-cloning-conversion/outputs/skill-voice-cloner.md"
},
{
"kind": "skill",
"name": "music-designer",
"description": "Pick a music-generation model, license strategy, length plan, and disclosure metadata for a deployment.",
"tags": [
"music-generation",
"musicgen",
"stable-audio",
"suno",
"licensing"
],
"phase": 6,
"lesson": 9,
"lessonPath": "phases/06-speech-and-audio/09-music-generation",
"file": "phases/06-speech-and-audio/09-music-generation/outputs/skill-music-designer.md"
},
{
"kind": "skill",
"name": "alm-picker",
"description": "Pick an audio-language model, benchmark subset, output modality (text vs speech), and guardrails for an audio-understanding task.",
"tags": [
"alm",
"lalm",
"qwen-omni",
"audio-flamingo",
"gemini-audio",
"mmau"
],
"phase": 6,
"lesson": 10,
"lessonPath": "phases/06-speech-and-audio/10-audio-language-models",
"file": "phases/06-speech-and-audio/10-audio-language-models/outputs/skill-alm-picker.md"
},
{
"kind": "skill",
"name": "realtime-voice-pipeline",
"description": "Pick transport, VAD, streaming STT, LLM, streaming TTS, and orchestration for a target end-to-end latency.",
"tags": [
"voice-agent",
"livekit",
"pipecat",
"silero",
"streaming",
"latency"
],
"phase": 6,
"lesson": 11,
"lessonPath": "phases/06-speech-and-audio/11-real-time-audio-processing",
"file": "phases/06-speech-and-audio/11-real-time-audio-processing/outputs/skill-realtime-pipeline.md"
},
{
"kind": "skill",
"name": "voice-assistant-architect",
"description": "Produce a full-stack voice-assistant spec — components, latency budget, observability, compliance — for a given workload.",
"tags": [
"voice-assistant",
"architecture",
"livekit",
"pipecat",
"compliance"
],
"phase": 6,
"lesson": 12,
"lessonPath": "phases/06-speech-and-audio/12-voice-assistant-pipeline",
"file": "phases/06-speech-and-audio/12-voice-assistant-pipeline/outputs/skill-voice-assistant-architect.md"
},
{
"kind": "skill",
"name": "codec-picker",
"description": "Pick a neural audio codec (EnCodec / DAC / SNAC / Mimi) for a given generative or compression task.",
"tags": [
"codec",
"encodec",
"dac",
"snac",
"mimi",
"rvq",
"semantic-tokens"
],
"phase": 6,
"lesson": 13,
"lessonPath": "phases/06-speech-and-audio/13-neural-audio-codecs",
"file": "phases/06-speech-and-audio/13-neural-audio-codecs/outputs/skill-codec-picker.md"
},
{
"kind": "skill",
"name": "vad-tuner",
"description": "Pick VAD model, threshold, silence hangover, pre-roll, and turn-detection strategy for a voice agent.",
"tags": [
"vad",
"silero",
"cobra",
"turn-detection",
"flush-trick"
],
"phase": 6,
"lesson": 14,
"lessonPath": "phases/06-speech-and-audio/14-voice-activity-detection-turn-taking",
"file": "phases/06-speech-and-audio/14-voice-activity-detection-turn-taking/outputs/skill-vad-tuner.md"
},
{
"kind": "skill",
"name": "duplex-pipeline",
"description": "Pick full-duplex (Moshi) vs pipeline (VAD + STT + LLM + TTS) architecture for a voice-agent workload.",
"tags": [
"moshi",
"hibiki",
"full-duplex",
"voice-agent",
"streaming"
],
"phase": 6,
"lesson": 15,
"lessonPath": "phases/06-speech-and-audio/15-streaming-speech-to-speech-moshi-hibiki",
"file": "phases/06-speech-and-audio/15-streaming-speech-to-speech-moshi-hibiki/outputs/skill-duplex-pipeline.md"
},
{
"kind": "skill",
"name": "spoof-defender",
"description": "Pick detection model, watermark, provenance manifest, and operational playbook for a voice-generation / voice-auth deployment.",
"tags": [
"anti-spoofing",
"watermark",
"audioseal",
"asvspoof",
"c2pa",
"voice-fraud"
],
"phase": 6,
"lesson": 16,
"lessonPath": "phases/06-speech-and-audio/16-anti-spoofing-audio-watermarking",
"file": "phases/06-speech-and-audio/16-anti-spoofing-audio-watermarking/outputs/skill-spoof-defender.md"
},
{
"kind": "skill",
"name": "audio-evaluator",
"description": "Pick metrics, benchmarks, normalization rules, and reporting format for any audio model release.",
"tags": [
"evaluation",
"wer",
"mos",
"utmos",
"eer",
"der",
"fad",
"mmau",
"leaderboard"
],
"phase": 6,
"lesson": 17,
"lessonPath": "phases/06-speech-and-audio/17-audio-evaluation-metrics",
"file": "phases/06-speech-and-audio/17-audio-evaluation-metrics/outputs/skill-audio-evaluator.md"
},
{
"kind": "skill",
"name": "sequence-architecture-picker",
"description": "Pick sequence architecture (RNN, transformer, SSM, hybrid) given length, throughput, and training budget.",
"tags": [
"transformers",
"architecture",
"rnn",
"ssm"
],
"phase": 7,
"lesson": 1,
"lessonPath": "phases/07-transformers-deep-dive/01-why-transformers",
"file": "phases/07-transformers-deep-dive/01-why-transformers/outputs/skill-architecture-picker.md"
},
{
"kind": "prompt",
"name": "prompt-attention-explainer",
"description": "Explain the attention mechanism through the database lookup analogy",
"tags": [],
"phase": 7,
"lesson": 2,
"lessonPath": "phases/07-transformers-deep-dive/02-self-attention-from-scratch",
"file": "phases/07-transformers-deep-dive/02-self-attention-from-scratch/outputs/prompt-attention-explainer.md"
},
{
"kind": "skill",
"name": "mha-configurator",
"description": "Recommend head count, KV-head count, and projection strategy (MHA / MQA / GQA / MLA) for a new transformer.",
"tags": [
"transformers",
"attention",
"mha",
"gqa"
],
"phase": 7,
"lesson": 3,
"lessonPath": "phases/07-transformers-deep-dive/03-multi-head-attention",
"file": "phases/07-transformers-deep-dive/03-multi-head-attention/outputs/skill-mha-configurator.md"
},
{
"kind": "skill",
"name": "positional-encoding-picker",
"description": "Pick positional encoding (RoPE, ALiBi, sinusoidal) + scaling strategy given context length and training budget.",
"tags": [
"transformers",
"positional-encoding",
"rope",
"alibi"
],
"phase": 7,
"lesson": 4,
"lessonPath": "phases/07-transformers-deep-dive/04-positional-encoding",
"file": "phases/07-transformers-deep-dive/04-positional-encoding/outputs/skill-positional-encoding-picker.md"
},
{
"kind": "skill",
"name": "transformer-block-reviewer",
"description": "Review a transformer block implementation against 2026 defaults and flag drift.",
"tags": [
"transformers",
"architecture",
"review"
],
"phase": 7,
"lesson": 5,
"lessonPath": "phases/07-transformers-deep-dive/05-full-transformer",
"file": "phases/07-transformers-deep-dive/05-full-transformer/outputs/skill-transformer-block-reviewer.md"
},
{
"kind": "skill",
"name": "bert-finetuner",
"description": "Scope a BERT fine-tune for a new classification, extraction, or retrieval task.",
"tags": [
"bert",
"fine-tuning",
"nlp"
],
"phase": 7,
"lesson": 6,
"lessonPath": "phases/07-transformers-deep-dive/06-bert-masked-language-modeling",
"file": "phases/07-transformers-deep-dive/06-bert-masked-language-modeling/outputs/skill-bert-finetuner.md"
},
{
"kind": "skill",
"name": "sampling-tuner",
"description": "Pick decoding strategy (greedy / temperature / top-k / top-p / min-p / speculative) for a given generation task.",
"tags": [
"gpt",
"sampling",
"decoding",
"inference"
],
"phase": 7,
"lesson": 7,
"lessonPath": "phases/07-transformers-deep-dive/07-gpt-causal-language-modeling",
"file": "phases/07-transformers-deep-dive/07-gpt-causal-language-modeling/outputs/skill-sampling-tuner.md"
},
{
"kind": "skill",
"name": "seq2seq-picker",
"description": "Choose encoder-decoder vs decoder-only for a new sequence-to-sequence task.",
"tags": [
"transformers",
"t5",
"bart",
"seq2seq"
],
"phase": 7,
"lesson": 8,
"lessonPath": "phases/07-transformers-deep-dive/08-t5-bart-encoder-decoder",
"file": "phases/07-transformers-deep-dive/08-t5-bart-encoder-decoder/outputs/skill-seq2seq-picker.md"
},
{
"kind": "skill",
"name": "vit-configurator",
"description": "Pick a ViT variant, patch size, and pretraining source for a new vision task.",
"tags": [
"transformers",
"vit",
"vision"
],
"phase": 7,
"lesson": 9,
"lessonPath": "phases/07-transformers-deep-dive/09-vision-transformers",
"file": "phases/07-transformers-deep-dive/09-vision-transformers/outputs/skill-vit-configurator.md"
},
{
"kind": "skill",
"name": "asr-configurator",
"description": "Pick an ASR model (Whisper variant / Moonshine / faster-whisper) and decoding parameters for a new speech pipeline.",
"tags": [
"transformers",
"whisper",
"asr",
"speech"
],
"phase": 7,
"lesson": 10,
"lessonPath": "phases/07-transformers-deep-dive/10-audio-transformers-whisper",
"file": "phases/07-transformers-deep-dive/10-audio-transformers-whisper/outputs/skill-asr-configurator.md"
},
{
"kind": "skill",
"name": "moe-configurator",
"description": "Pick expert count, top-k, balancing strategy, and shared-expert layout for a new MoE transformer.",
"tags": [
"transformers",
"moe",
"mixture-of-experts",
"scaling"
],
"phase": 7,
"lesson": 11,
"lessonPath": "phases/07-transformers-deep-dive/11-mixture-of-experts",
"file": "phases/07-transformers-deep-dive/11-mixture-of-experts/outputs/skill-moe-configurator.md"
},
{
"kind": "skill",
"name": "inference-optimizer",
"description": "Pick attention implementation, KV cache strategy, quantization, and speculative decoding for a new inference deployment.",
"tags": [
"transformers",
"inference",
"flash-attention",
"kv-cache"
],
"phase": 7,
"lesson": 12,
"lessonPath": "phases/07-transformers-deep-dive/12-kv-cache-flash-attention",
"file": "phases/07-transformers-deep-dive/12-kv-cache-flash-attention/outputs/skill-inference-optimizer.md"
},
{
"kind": "skill",
"name": "training-budget-estimator",
"description": "Estimate (N, D, hours, GPU count) for a new transformer training run given compute budget and deployment constraints.",
"tags": [
"scaling-laws",
"training",
"chinchilla"
],
"phase": 7,
"lesson": 13,
"lessonPath": "phases/07-transformers-deep-dive/13-scaling-laws",
"file": "phases/07-transformers-deep-dive/13-scaling-laws/outputs/skill-training-budget-estimator.md"
},
{
"kind": "skill",
"name": "transformer-review",
"description": "Review a transformer-from-scratch implementation against the 13 Phase 7 lessons.",
"tags": [
"transformers",
"review",
"capstone"
],
"phase": 7,
"lesson": 14,
"lessonPath": "phases/07-transformers-deep-dive/14-build-a-transformer-capstone",
"file": "phases/07-transformers-deep-dive/14-build-a-transformer-capstone/outputs/skill-transformer-review.md"
},
{
"kind": "skill",
"name": "attention-variant-picker",
"description": "Pick a full / sliding-window / sparse / differential attention topology for a new model given context length, retrieval demands, and compute profile.",
"tags": [
"attention",
"transformer",
"long-context",
"inference",
"memory"
],
"phase": 7,
"lesson": 15,
"lessonPath": "phases/07-transformers-deep-dive/15-attention-variants",
"file": "phases/07-transformers-deep-dive/15-attention-variants/outputs/skill-attention-variant-picker.md"
},
{
"kind": "skill",
"name": "spec-decode-picker",
"description": "Pick a speculative decoding strategy (vanilla / Medusa / EAGLE / lookahead) and tuning parameters for a new LLM inference workload.",
"tags": [
"inference",
"decoding",
"latency",
"speculative",
"optimization"
],
"phase": 7,
"lesson": 16,
"lessonPath": "phases/07-transformers-deep-dive/16-speculative-decoding",
"file": "phases/07-transformers-deep-dive/16-speculative-decoding/outputs/skill-spec-decode-picker.md"
},
{
"kind": "skill",
"name": "generative-model-chooser",
"description": "Pick a generative-model family, backbone, and hosted alternative for a given task and budget.",
"tags": [
"generative",
"taxonomy"
],
"phase": 8,
"lesson": 1,
"lessonPath": "phases/08-generative-ai/01-generative-models-taxonomy-history",
"file": "phases/08-generative-ai/01-generative-models-taxonomy-history/outputs/skill-model-chooser.md"
},
{
"kind": "skill",
"name": "vae-trainer",
"description": "Specify VAE architecture, latent size, beta schedule, and eval plan for a given dataset and downstream use.",
"tags": [
"vae",
"latent",
"generative"
],
"phase": 8,
"lesson": 2,
"lessonPath": "phases/08-generative-ai/02-autoencoders-vae",
"file": "phases/08-generative-ai/02-autoencoders-vae/outputs/skill-vae-trainer.md"
},
{
"kind": "skill",
"name": "gan-debugger",
"description": "Diagnose failing GAN training from loss curves and sample grids; prescribe one-line fixes.",
"tags": [
"gan",
"adversarial",
"debugging"
],
"phase": 8,
"lesson": 3,
"lessonPath": "phases/08-generative-ai/03-gans-generator-discriminator",
"file": "phases/08-generative-ai/03-gans-generator-discriminator/outputs/skill-gan-debugger.md"
},
{
"kind": "skill",
"name": "img2img-chooser",
"description": "Pick an image-to-image approach given paired vs unpaired data, domain specificity, and latency budget.",
"tags": [
"pix2pix",
"img2img",
"conditional"
],
"phase": 8,
"lesson": 4,
"lessonPath": "phases/08-generative-ai/04-conditional-gans-pix2pix",
"file": "phases/08-generative-ai/04-conditional-gans-pix2pix/outputs/skill-img2img-chooser.md"
},
{
"kind": "skill",
"name": "stylegan-inversion",
"description": "Choose an inversion and editing pipeline for a pretrained StyleGAN over a real photo.",
"tags": [
"stylegan",
"inversion",
"editing"
],
"phase": 8,
"lesson": 5,
"lessonPath": "phases/08-generative-ai/05-stylegan",
"file": "phases/08-generative-ai/05-stylegan/outputs/skill-stylegan-inversion.md"
},
{
"kind": "skill",
"name": "diffusion-trainer",
"description": "Configure a diffusion training run: schedule, prediction target, sampler, and eval plan.",
"tags": [
"diffusion",
"ddpm",
"training"
],
"phase": 8,
"lesson": 6,
"lessonPath": "phases/08-generative-ai/06-diffusion-ddpm-from-scratch",
"file": "phases/08-generative-ai/06-diffusion-ddpm-from-scratch/outputs/skill-diffusion-trainer.md"
},
{
"kind": "skill",
"name": "sd-prompter",
"description": "Configure Stable Diffusion / Flux inference for a given prompt, style, and quality bar.",
"tags": [
"stable-diffusion",
"flux",
"latent-diffusion"
],
"phase": 8,
"lesson": 7,
"lessonPath": "phases/08-generative-ai/07-latent-diffusion-stable-diffusion",
"file": "phases/08-generative-ai/07-latent-diffusion-stable-diffusion/outputs/skill-sd-prompter.md"
},
{
"kind": "skill",
"name": "sd-toolkit-composer",
"description": "Compose ControlNets, LoRAs, and IP-Adapters on top of an SD / Flux base for a given set of inputs.",
"tags": [
"controlnet",
"lora",
"ip-adapter",
"diffusion"
],
"phase": 8,
"lesson": 8,
"lessonPath": "phases/08-generative-ai/08-controlnet-lora-conditioning",
"file": "phases/08-generative-ai/08-controlnet-lora-conditioning/outputs/skill-sd-toolkit-composer.md"
},
{
"kind": "skill",
"name": "editing-pipeline",
"description": "Plan an image-editing pipeline from source + edit description to a ready-to-ship output.",
"tags": [
"inpaint",
"outpaint",
"edit",
"sam"
],
"phase": 8,
"lesson": 9,
"lessonPath": "phases/08-generative-ai/09-inpainting-outpainting-editing",
"file": "phases/08-generative-ai/09-inpainting-outpainting-editing/outputs/skill-editing-pipeline.md"
},
{
"kind": "skill",
"name": "video-brief",
"description": "Translate a video brief into a model + prompt + shot plan for a 2026 video generator.",
"tags": [
"video",
"diffusion",
"sora",
"veo",
"kling"
],
"phase": 8,
"lesson": 10,
"lessonPath": "phases/08-generative-ai/10-video-generation",
"file": "phases/08-generative-ai/10-video-generation/outputs/skill-video-brief.md"
},
{
"kind": "skill",
"name": "audio-brief",
"description": "Translate an audio brief into a model + prompt + eval plan across TTS, music, and SFX.",
"tags": [
"audio",
"tts",
"music",
"sfx",
"codec"
],
"phase": 8,
"lesson": 11,
"lessonPath": "phases/08-generative-ai/11-audio-generation",
"file": "phases/08-generative-ai/11-audio-generation/outputs/skill-audio-brief.md"
},
{
"kind": "skill",
"name": "3d-pipeline",
"description": "Choose a 3D generation or reconstruction pipeline given input type, output format, and use case.",
"tags": [
"3d",
"gaussian-splatting",
"nerf",
"mesh"
],
"phase": 8,
"lesson": 12,
"lessonPath": "phases/08-generative-ai/12-3d-generation",
"file": "phases/08-generative-ai/12-3d-generation/outputs/skill-3d-pipeline.md"
},
{
"kind": "skill",
"name": "fm-tuner",
"description": "Convert a diffusion training plan into a flow-matching / rectified-flow config.",
"tags": [
"flow-matching",
"rectified-flow",
"diffusion"
],
"phase": 8,
"lesson": 13,
"lessonPath": "phases/08-generative-ai/13-flow-matching-rectified-flows",
"file": "phases/08-generative-ai/13-flow-matching-rectified-flows/outputs/skill-fm-tuner.md"
},
{
"kind": "skill",
"name": "eval-report",
"description": "Plan a full generative-model evaluation: sample quality, adherence, preference, failure audit.",
"tags": [
"evaluation",
"fid",
"clip",
"elo"
],
"phase": 8,
"lesson": 14,
"lessonPath": "phases/08-generative-ai/14-evaluation-fid-clip-score",
"file": "phases/08-generative-ai/14-evaluation-fid-clip-score/outputs/skill-eval-report.md"
},
{
"kind": "skill",
"name": "var-tokenizer-designer",
"description": "Design a multi-scale residual VQ tokenizer for next-scale visual autoregressive image generation.",
"tags": [
"var",
"next-scale-prediction",
"vq-vae",
"residual-vq",
"image-generation",
"tokenizer"
],
"phase": 8,
"lesson": 19,
"lessonPath": "phases/08-generative-ai/19-visual-autoregressive-var",
"file": "phases/08-generative-ai/19-visual-autoregressive-var/outputs/skill-var-tokenizer-designer.md"
},
{
"kind": "skill",
"name": "mdp-modeler",
"description": "Given a task description, produce a Markov Decision Process spec and flag formulation risks before training.",
"tags": [
"rl",
"mdp",
"modeling"
],
"phase": 9,
"lesson": 1,
"lessonPath": "phases/09-reinforcement-learning/01-mdps-states-actions-rewards",
"file": "phases/09-reinforcement-learning/01-mdps-states-actions-rewards/outputs/skill-mdp-modeler.md"
},
{
"kind": "skill",
"name": "dp-solver",
"description": "Solve a small tabular MDP exactly via policy iteration or value iteration. Report convergence behavior.",
"tags": [
"rl",
"dynamic-programming",
"bellman"
],
"phase": 9,
"lesson": 2,
"lessonPath": "phases/09-reinforcement-learning/02-dynamic-programming",
"file": "phases/09-reinforcement-learning/02-dynamic-programming/outputs/skill-dp-solver.md"
},
{
"kind": "skill",
"name": "mc-evaluator",
"description": "Evaluate a policy via Monte Carlo rollouts and produce a convergence report with DP-comparison if available.",
"tags": [
"rl",
"monte-carlo",
"evaluation"
],
"phase": 9,
"lesson": 3,
"lessonPath": "phases/09-reinforcement-learning/03-monte-carlo-methods",
"file": "phases/09-reinforcement-learning/03-monte-carlo-methods/outputs/skill-mc-evaluator.md"
},
{
"kind": "skill",
"name": "td-agent",
"description": "Pick between Q-learning, SARSA, Expected SARSA for a tabular or small-feature RL task.",
"tags": [
"rl",
"td-learning",
"q-learning",
"sarsa"
],
"phase": 9,
"lesson": 4,
"lessonPath": "phases/09-reinforcement-learning/04-q-learning-sarsa",
"file": "phases/09-reinforcement-learning/04-q-learning-sarsa/outputs/skill-td-agent.md"
},
{
"kind": "skill",
"name": "dqn-trainer",
"description": "Produce a DQN training config (buffer, target sync, ε schedule, reward clipping) for a discrete-action RL task.",
"tags": [
"rl",
"dqn",
"deep-rl"
],
"phase": 9,
"lesson": 5,
"lessonPath": "phases/09-reinforcement-learning/05-dqn",
"file": "phases/09-reinforcement-learning/05-dqn/outputs/skill-dqn-trainer.md"
},
{
"kind": "skill",
"name": "policy-gradient-trainer",
"description": "Produce a REINFORCE / actor-critic / PPO training config for a given task and diagnose variance issues.",
"tags": [
"rl",
"policy-gradient",
"reinforce"
],
"phase": 9,
"lesson": 6,
"lessonPath": "phases/09-reinforcement-learning/06-policy-gradients-reinforce",
"file": "phases/09-reinforcement-learning/06-policy-gradients-reinforce/outputs/skill-policy-gradient-trainer.md"
},
{
"kind": "skill",
"name": "actor-critic-trainer",
"description": "Produce an A2C / A3C / GAE configuration for a given environment, with advantage estimation and loss weights specified.",
"tags": [
"rl",
"actor-critic",
"gae"
],
"phase": 9,
"lesson": 7,
"lessonPath": "phases/09-reinforcement-learning/07-actor-critic-a2c-a3c",
"file": "phases/09-reinforcement-learning/07-actor-critic-a2c-a3c/outputs/skill-actor-critic-trainer.md"
},
{
"kind": "skill",
"name": "ppo-trainer",
"description": "Produce a PPO training config and a diagnostic plan for a given environment.",
"tags": [
"rl",
"ppo",
"policy-gradient"
],
"phase": 9,
"lesson": 8,
"lessonPath": "phases/09-reinforcement-learning/08-ppo",
"file": "phases/09-reinforcement-learning/08-ppo/outputs/skill-ppo-trainer.md"
},
{
"kind": "skill",
"name": "rlhf-architect",
"description": "Design an RLHF / DPO / GRPO alignment pipeline for a language model, including RM, KL, and data strategy.",
"tags": [
"rl",
"rlhf",
"alignment",
"llm"
],
"phase": 9,
"lesson": 9,
"lessonPath": "phases/09-reinforcement-learning/09-reward-modeling-rlhf",
"file": "phases/09-reinforcement-learning/09-reward-modeling-rlhf/outputs/skill-rlhf-architect.md"
},
{
"kind": "skill",
"name": "marl-architect",
"description": "Pick the right multi-agent RL regime (IPPO, CTDE, self-play, league) for a given task.",
"tags": [
"rl",
"multi-agent",
"marl",
"self-play"
],
"phase": 9,
"lesson": 10,
"lessonPath": "phases/09-reinforcement-learning/10-multi-agent-rl",
"file": "phases/09-reinforcement-learning/10-multi-agent-rl/outputs/skill-marl-architect.md"
},
{
"kind": "skill",
"name": "sim2real-planner",
"description": "Plan a sim-to-real transfer pipeline for a given robot + task, covering DR, SI, and safety.",
"tags": [
"rl",
"sim2real",
"robotics",
"domain-randomization"
],
"phase": 9,
"lesson": 11,
"lessonPath": "phases/09-reinforcement-learning/11-sim-to-real-transfer",
"file": "phases/09-reinforcement-learning/11-sim-to-real-transfer/outputs/skill-sim2real-planner.md"
},
{
"kind": "skill",
"name": "game-rl-designer",
"description": "Design a game-RL or reasoning-RL training pipeline (AlphaZero / MuZero / GRPO) for a given domain.",
"tags": [
"rl",
"alphazero",
"muzero",
"grpo",
"self-play"
],
"phase": 9,
"lesson": 12,
"lessonPath": "phases/09-reinforcement-learning/12-rl-for-games",
"file": "phases/09-reinforcement-learning/12-rl-for-games/outputs/skill-game-rl-designer.md"
},
{
"kind": "prompt",
"name": "prompt-tokenizer-analyzer",
"description": "Analyze tokenization efficiency for a given text across different models and tokenizer types",
"tags": [],
"phase": 10,
"lesson": 1,
"lessonPath": "phases/10-llms-from-scratch/01-tokenizers",
"file": "phases/10-llms-from-scratch/01-tokenizers/outputs/prompt-tokenizer-analyzer.md"
},
{
"kind": "skill",
"name": "skill-tokenizer",
"description": "Choosing and building tokenizers for LLM projects",
"tags": [
"tokenizer",
"bpe",
"wordpiece",
"sentencepiece",
"llm",
"nlp"
],
"phase": 10,
"lesson": 1,
"lessonPath": "phases/10-llms-from-scratch/01-tokenizers",
"file": "phases/10-llms-from-scratch/01-tokenizers/outputs/skill-tokenizer.md"
},
{
"kind": "prompt",
"name": "prompt-tokenizer-builder",
"description": "Build and debug production-quality tokenizers for LLM projects",
"tags": [
"tokenizer",
"bpe",
"byte-level",
"special-tokens",
"chat-template",
"multilingual"
],
"phase": 10,
"lesson": 2,
"lessonPath": "phases/10-llms-from-scratch/02-building-a-tokenizer",
"file": "phases/10-llms-from-scratch/02-building-a-tokenizer/outputs/prompt-tokenizer-builder.md"
},
{
"kind": "prompt",
"name": "prompt-data-quality-checker",
"description": "Validate and debug data quality in LLM pre-training pipelines",
"tags": [
"data-pipeline",
"deduplication",
"quality-filter",
"pre-training",
"llm",
"data-cleaning"
],
"phase": 10,
"lesson": 3,
"lessonPath": "phases/10-llms-from-scratch/03-data-pipelines",
"file": "phases/10-llms-from-scratch/03-data-pipelines/outputs/prompt-data-quality-checker.md"
},
{
"kind": "prompt",
"name": "prompt-gpt-architecture-analyzer",
"description": "Analyze architecture choices in any GPT-style transformer model",
"tags": [
"gpt",
"transformer",
"architecture",
"attention",
"kv-cache",
"scaling",
"pre-training"
],
"phase": 10,
"lesson": 4,
"lessonPath": "phases/10-llms-from-scratch/04-pre-training-mini-gpt",
"file": "phases/10-llms-from-scratch/04-pre-training-mini-gpt/outputs/prompt-gpt-architecture-analyzer.md"
},
{
"kind": "prompt",
"name": "prompt-distributed-training-planner",
"description": "Plan a distributed training run given model size and available hardware",
"tags": [
"distributed-training",
"fsdp",
"deepspeed",
"tensor-parallelism",
"pipeline-parallelism",
"scaling"
],
"phase": 10,
"lesson": 5,
"lessonPath": "phases/10-llms-from-scratch/05-scaling-distributed",
"file": "phases/10-llms-from-scratch/05-scaling-distributed/outputs/prompt-distributed-training-planner.md"
},
{
"kind": "prompt",
"name": "prompt-sft-data-curator",
"description": "Design and curate instruction datasets for supervised fine-tuning",
"tags": [
"sft",
"instruction-tuning",
"fine-tuning",
"data-curation",
"alignment"
],
"phase": 10,
"lesson": 6,
"lessonPath": "phases/10-llms-from-scratch/06-instruction-tuning-sft",
"file": "phases/10-llms-from-scratch/06-instruction-tuning-sft/outputs/prompt-sft-data-curator.md"
},
{
"kind": "prompt",
"name": "prompt-reward-model-designer",
"description": "Design reward model training pipelines for RLHF alignment",
"tags": [
"rlhf",
"reward-model",
"ppo",
"alignment",
"human-feedback",
"preference-learning"
],
"phase": 10,
"lesson": 7,
"lessonPath": "phases/10-llms-from-scratch/07-rlhf",
"file": "phases/10-llms-from-scratch/07-rlhf/outputs/prompt-reward-model-designer.md"
},
{
"kind": "prompt",
"name": "prompt-alignment-method-selector",
"description": "Choose the right alignment method (SFT, RLHF, DPO, KTO, ORPO, SimPO) for your use case",
"tags": [
"alignment",
"dpo",
"rlhf",
"kto",
"orpo",
"simpo",
"preference-optimization",
"fine-tuning"
],
"phase": 10,
"lesson": 8,
"lessonPath": "phases/10-llms-from-scratch/08-dpo",
"file": "phases/10-llms-from-scratch/08-dpo/outputs/prompt-alignment-method-selector.md"
},
{
"kind": "skill",
"name": "self-improvement-auditor",
"description": "Audit a proposed self-improvement or constitutional AI pipeline before it runs at scale.",
"tags": [
"alignment",
"cai",
"grpo",
"rlhf",
"self-improvement",
"reward-hacking"
],
"phase": 10,
"lesson": 9,
"lessonPath": "phases/10-llms-from-scratch/09-constitutional-ai-self-improvement",
"file": "phases/10-llms-from-scratch/09-constitutional-ai-self-improvement/outputs/skill-self-improvement-auditor.md"
},
{
"kind": "prompt",
"name": "prompt-eval-designer",
"description": "Design a custom evaluation suite for any LLM task, including test cases, scoring functions, and pass/fail thresholds",
"tags": [],
"phase": 10,
"lesson": 10,
"lessonPath": "phases/10-llms-from-scratch/10-evaluation",
"file": "phases/10-llms-from-scratch/10-evaluation/outputs/prompt-eval-designer.md"
},
{
"kind": "skill",
"name": "skill-llm-evaluation",
"description": "Decision framework for choosing the right LLM evaluation strategy based on task type, budget, and requirements",
"tags": [
"evaluation",
"evals",
"benchmarks",
"llm-as-judge",
"elo",
"metrics"
],
"phase": 10,
"lesson": 10,
"lessonPath": "phases/10-llms-from-scratch/10-evaluation",
"file": "phases/10-llms-from-scratch/10-evaluation/outputs/skill-llm-evaluation.md"
},
{
"kind": "skill",
"name": "skill-quantization",
"description": "Choose the right quantization strategy for deploying LLMs based on hardware, quality, and latency constraints",
"tags": [
"quantization",
"inference",
"deployment",
"optimization",
"fp8",
"int4",
"int8",
"gptq",
"awq",
"gguf"
],
"phase": 10,
"lesson": 11,
"lessonPath": "phases/10-llms-from-scratch/11-quantization",
"file": "phases/10-llms-from-scratch/11-quantization/outputs/skill-quantization.md"
},
{
"kind": "skill",
"name": "skill-inference-optimization",
"description": "Diagnose and optimize LLM inference serving throughput, latency, and cost",
"tags": [
"inference",
"kv-cache",
"batching",
"speculative-decoding",
"vllm",
"optimization"
],
"phase": 10,
"lesson": 12,
"lessonPath": "phases/10-llms-from-scratch/12-inference-optimization",
"file": "phases/10-llms-from-scratch/12-inference-optimization/outputs/skill-inference-optimization.md"
},
{
"kind": "skill",
"name": "llm-pipeline-reviewer",
"description": "Review an end-to-end LLM training pipeline manifest before a multi-million-dollar run.",
"tags": [
"pipeline",
"training",
"manifest",
"eval-gate",
"cost",
"rollback"
],
"phase": 10,
"lesson": 13,
"lessonPath": "phases/10-llms-from-scratch/13-building-complete-llm-pipeline",
"file": "phases/10-llms-from-scratch/13-building-complete-llm-pipeline/outputs/skill-llm-pipeline-reviewer.md"
},
{
"kind": "skill",
"name": "open-model-picker",
"description": "Pick an open LLM family, quantization, and inference stack for a given deployment target.",
"tags": [
"open-models",
"llama",
"deepseek",
"mixtral",
"qwen",
"gemma",
"moe",
"gqa",
"mla",
"quantization"
],
"phase": 10,
"lesson": 14,
"lessonPath": "phases/10-llms-from-scratch/14-open-models-architecture-walkthroughs",
"file": "phases/10-llms-from-scratch/14-open-models-architecture-walkthroughs/outputs/skill-open-model-picker.md"
},
{
"kind": "skill",
"name": "eagle3-tuner",
"description": "Pick and tune a speculative decoding strategy (vanilla / Medusa / EAGLE-1/2/3 / lookahead) for a new inference workload.",
"tags": [
"speculative-decoding",
"eagle",
"eagle-3",
"medusa",
"inference",
"vllm",
"sglang",
"tensorrt-llm"
],
"phase": 10,
"lesson": 15,
"lessonPath": "phases/10-llms-from-scratch/15-speculative-decoding-eagle3",
"file": "phases/10-llms-from-scratch/15-speculative-decoding-eagle3/outputs/skill-eagle3-tuner.md"
},
{
"kind": "skill",
"name": "diff-attention-integrator",
"description": "Integration plan for adding Differential Attention V2 to a new pre-training run or LoRA fine-tune.",
"tags": [
"differential-attention",
"diff-transformer",
"long-context",
"flash-attention",
"pre-training",
"lora"
],
"phase": 10,
"lesson": 16,
"lessonPath": "phases/10-llms-from-scratch/16-differential-attention-v2",
"file": "phases/10-llms-from-scratch/16-differential-attention-v2/outputs/skill-diff-attention-integrator.md"
},
{
"kind": "skill",
"name": "nsa-integrator",
"description": "Integration plan for Native Sparse Attention in a long-context pre-training run.",
"tags": [
"nsa",
"sparse-attention",
"long-context",
"pre-training",
"kernel-aligned",
"deepseek"
],
"phase": 10,
"lesson": 17,
"lessonPath": "phases/10-llms-from-scratch/17-native-sparse-attention",
"file": "phases/10-llms-from-scratch/17-native-sparse-attention/outputs/skill-nsa-integrator.md"
},
{
"kind": "skill",
"name": "mtp-planner",
"description": "Plan a multi-token prediction integration for a new pre-training run.",
"tags": [
"mtp",
"multi-token-prediction",
"deepseek-v3",
"pre-training",
"speculative-decoding"
],
"phase": 10,
"lesson": 18,
"lessonPath": "phases/10-llms-from-scratch/18-multi-token-prediction",
"file": "phases/10-llms-from-scratch/18-multi-token-prediction/outputs/skill-mtp-planner.md"
},
{
"kind": "skill",
"name": "dualpipe-planner",
"description": "Plan a pipeline parallelism strategy (1F1B, Zero Bubble, DualPipe, DualPipeV) for a training cluster.",
"tags": [
"pipeline-parallelism",
"dualpipe",
"dualpipev",
"zero-bubble",
"expert-parallelism",
"distributed-training"
],
"phase": 10,
"lesson": 19,
"lessonPath": "phases/10-llms-from-scratch/19-dualpipe-parallelism",
"file": "phases/10-llms-from-scratch/19-dualpipe-parallelism/outputs/skill-dualpipe-planner.md"
},
{
"kind": "skill",
"name": "deepseek-v3-reader",
"description": "Read a DeepSeek-family config and produce a component-by-component architecture analysis.",
"tags": [
"deepseek-v3",
"deepseek-r1",
"mla",
"moe",
"mtp",
"dualpipe",
"architecture"
],
"phase": 10,
"lesson": 20,
"lessonPath": "phases/10-llms-from-scratch/20-deepseek-v3-walkthrough",
"file": "phases/10-llms-from-scratch/20-deepseek-v3-walkthrough/outputs/skill-deepseek-v3-reader.md"
},
{
"kind": "skill",
"name": "hybrid-picker",
"description": "Pick between pure Transformer, Jamba-style hybrid, and pure SSM for a given workload.",
"tags": [
"jamba",
"mamba",
"ssm",
"hybrid",
"long-context",
"memory-budget",
"architecture"
],
"phase": 10,
"lesson": 21,
"lessonPath": "phases/10-llms-from-scratch/21-jamba-hybrid-ssm-transformer",
"file": "phases/10-llms-from-scratch/21-jamba-hybrid-ssm-transformer/outputs/skill-hybrid-picker.md"
},
{
"kind": "skill",
"name": "parallel-inference-router",
"description": "Route a reasoning workload between voting, tree-of-thought, multi-agent, Hogwild!, and speculative decoding strategies.",
"tags": [
"parallel-inference",
"hogwild",
"speculative-decoding",
"tree-of-thought",
"multi-agent",
"reasoning"
],
"phase": 10,
"lesson": 22,
"lessonPath": "phases/10-llms-from-scratch/22-async-hogwild-inference",
"file": "phases/10-llms-from-scratch/22-async-hogwild-inference/outputs/skill-parallel-inference-router.md"
},
{
"kind": "skill",
"name": "speculative-tuning",
"description": "Profile a decode workload and pick draft model, draft length K, temperature gate, and fallback policy for speculative decoding.",
"tags": [
"speculative-decoding",
"draft-model",
"alpha",
"throughput",
"inference",
"decode-latency"
],
"phase": 10,
"lesson": 25,
"lessonPath": "phases/10-llms-from-scratch/25-speculative-decoding",
"file": "phases/10-llms-from-scratch/25-speculative-decoding/outputs/skill-speculative-tuning.md"
},
{
"kind": "skill",
"name": "checkpointing-planner",
"description": "Choose an activation recomputation policy per layer (none / selective / full / offload) given a training config and HBM budget.",
"tags": [
"gradient-checkpointing",
"activation-recomputation",
"selective-checkpoint",
"fsdp-offload",
"training-memory"
],
"phase": 10,
"lesson": 34,
"lessonPath": "phases/10-llms-from-scratch/34-gradient-checkpointing",
"file": "phases/10-llms-from-scratch/34-gradient-checkpointing/outputs/skill-checkpointing-planner.md"
},
{
"kind": "prompt",
"name": "prompt-prompt-optimizer",
"description": "Takes a draft prompt and rewrites it using proven prompt engineering patterns for maximum effectiveness across models",
"tags": [],
"phase": 11,
"lesson": 1,
"lessonPath": "phases/11-llm-engineering/01-prompt-engineering",
"file": "phases/11-llm-engineering/01-prompt-engineering/outputs/prompt-prompt-optimizer.md"
},
{
"kind": "skill",
"name": "skill-prompt-patterns",
"description": "Decision framework for choosing the right prompt pattern based on task type, reliability requirements, and target model",
"tags": [
"prompt-engineering",
"patterns",
"llm",
"temperature",
"cross-model",
"few-shot",
"chain-of-thought"
],
"phase": 11,
"lesson": 1,
"lessonPath": "phases/11-llm-engineering/01-prompt-engineering",
"file": "phases/11-llm-engineering/01-prompt-engineering/outputs/skill-prompt-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-reasoning-chain",
"description": "Production-ready few-shot CoT prompt with self-consistency support for multi-step reasoning tasks",
"tags": [],
"phase": 11,
"lesson": 2,
"lessonPath": "phases/11-llm-engineering/02-few-shot-cot",
"file": "phases/11-llm-engineering/02-few-shot-cot/outputs/prompt-reasoning-chain.md"
},
{
"kind": "skill",
"name": "skill-cot-patterns",
"description": "Decision framework for choosing the right reasoning technique based on task complexity, accuracy requirements, and cost constraints",
"tags": [
"chain-of-thought",
"few-shot",
"self-consistency",
"tree-of-thought",
"react",
"reasoning",
"prompting"
],
"phase": 11,
"lesson": 2,
"lessonPath": "phases/11-llm-engineering/02-few-shot-cot",
"file": "phases/11-llm-engineering/02-few-shot-cot/outputs/skill-cot-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-structured-extractor",
"description": "Extract structured data from unstructured text given a JSON Schema definition",
"tags": [],
"phase": 11,
"lesson": 3,
"lessonPath": "phases/11-llm-engineering/03-structured-outputs",
"file": "phases/11-llm-engineering/03-structured-outputs/outputs/prompt-structured-extractor.md"
},
{
"kind": "skill",
"name": "skill-structured-outputs",
"description": "Decision framework for choosing the right structured output strategy based on provider, reliability, and complexity",
"tags": [
"structured-output",
"json",
"schema",
"constrained-decoding",
"pydantic",
"function-calling"
],
"phase": 11,
"lesson": 3,
"lessonPath": "phases/11-llm-engineering/03-structured-outputs",
"file": "phases/11-llm-engineering/03-structured-outputs/outputs/skill-structured-outputs.md"
},
{
"kind": "prompt",
"name": "prompt-embedding-advisor",
"description": "Choose embedding models, dimensions, and strategies for specific use cases",
"tags": [],
"phase": 11,
"lesson": 4,
"lessonPath": "phases/11-llm-engineering/04-embeddings",
"file": "phases/11-llm-engineering/04-embeddings/outputs/prompt-embedding-advisor.md"
},
{
"kind": "skill",
"name": "skill-embedding-patterns",
"description": "Production patterns for embeddings, vector search, and similarity",
"tags": [
"embeddings",
"vectors",
"similarity",
"search",
"chunking",
"quantization"
],
"phase": 11,
"lesson": 4,
"lessonPath": "phases/11-llm-engineering/04-embeddings",
"file": "phases/11-llm-engineering/04-embeddings/outputs/skill-embedding-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-context-optimizer",
"description": "Audit a context assembly strategy and recommend optimizations to reduce token waste and improve response quality",
"tags": [],
"phase": 11,
"lesson": 5,
"lessonPath": "phases/11-llm-engineering/05-context-engineering",
"file": "phases/11-llm-engineering/05-context-engineering/outputs/prompt-context-optimizer.md"
},
{
"kind": "skill",
"name": "skill-context-engineering",
"description": "Decision framework for designing context assembly pipelines based on task type, window size, and latency budget",
"tags": [
"context-engineering",
"context-window",
"rag",
"memory",
"tool-selection",
"lost-in-the-middle"
],
"phase": 11,
"lesson": 5,
"lessonPath": "phases/11-llm-engineering/05-context-engineering",
"file": "phases/11-llm-engineering/05-context-engineering/outputs/skill-context-engineering.md"
},
{
"kind": "prompt",
"name": "prompt-rag-architect",
"description": "Design RAG systems for specific use cases with concrete architecture decisions",
"tags": [],
"phase": 11,
"lesson": 6,
"lessonPath": "phases/11-llm-engineering/06-rag",
"file": "phases/11-llm-engineering/06-rag/outputs/prompt-rag-architect.md"
},
{
"kind": "skill",
"name": "skill-rag-pipeline",
"description": "Build and debug RAG pipelines from first principles",
"tags": [
"rag",
"retrieval",
"embeddings",
"vector-search",
"llm-engineering"
],
"phase": 11,
"lesson": 6,
"lessonPath": "phases/11-llm-engineering/06-rag",
"file": "phases/11-llm-engineering/06-rag/outputs/skill-rag-pipeline.md"
},
{
"kind": "prompt",
"name": "prompt-advanced-rag-debugger",
"description": "Diagnose and fix RAG quality issues across retrieval, generation, and evaluation",
"tags": [],
"phase": 11,
"lesson": 7,
"lessonPath": "phases/11-llm-engineering/07-advanced-rag",
"file": "phases/11-llm-engineering/07-advanced-rag/outputs/prompt-advanced-rag-debugger.md"
},
{
"kind": "skill",
"name": "skill-advanced-rag",
"description": "Build production-grade RAG with hybrid search, reranking, and evaluation",
"tags": [
"rag",
"hybrid-search",
"bm25",
"reranking",
"hyde",
"evaluation"
],
"phase": 11,
"lesson": 7,
"lessonPath": "phases/11-llm-engineering/07-advanced-rag",
"file": "phases/11-llm-engineering/07-advanced-rag/outputs/skill-advanced-rag.md"
},
{
"kind": "prompt",
"name": "prompt-lora-advisor",
"description": "Decide LoRA rank, target modules, and hyperparameters for a specific fine-tuning task",
"tags": [],
"phase": 11,
"lesson": 8,
"lessonPath": "phases/11-llm-engineering/08-fine-tuning-lora",
"file": "phases/11-llm-engineering/08-fine-tuning-lora/outputs/prompt-lora-advisor.md"
},
{
"kind": "skill",
"name": "skill-fine-tuning-guide",
"description": "Decision tree for when and how to fine-tune LLMs with LoRA and QLoRA",
"tags": [
"fine-tuning",
"lora",
"qlora",
"peft",
"llm-engineering"
],
"phase": 11,
"lesson": 8,
"lessonPath": "phases/11-llm-engineering/08-fine-tuning-lora",
"file": "phases/11-llm-engineering/08-fine-tuning-lora/outputs/skill-fine-tuning-guide.md"
},
{
"kind": "prompt",
"name": "prompt-tool-designer",
"description": "Design complete tool definitions (JSON Schema) for function calling from a natural language description",
"tags": [],
"phase": 11,
"lesson": 9,
"lessonPath": "phases/11-llm-engineering/09-function-calling",
"file": "phases/11-llm-engineering/09-function-calling/outputs/prompt-tool-designer.md"
},
{
"kind": "skill",
"name": "skill-function-calling-patterns",
"description": "Decision framework for implementing function calling in production -- tool design, error handling, security, and provider patterns",
"tags": [
"function-calling",
"tool-use",
"agents",
"mcp",
"security",
"openai",
"anthropic"
],
"phase": 11,
"lesson": 9,
"lessonPath": "phases/11-llm-engineering/09-function-calling",
"file": "phases/11-llm-engineering/09-function-calling/outputs/skill-function-calling-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-eval-designer",
"description": "Design tailored evaluation rubrics and test suites for LLM applications from a description of the use case",
"tags": [],
"phase": 11,
"lesson": 10,
"lessonPath": "phases/11-llm-engineering/10-evaluation",
"file": "phases/11-llm-engineering/10-evaluation/outputs/prompt-eval-designer.md"
},
{
"kind": "skill",
"name": "skill-eval-patterns",
"description": "Decision framework for choosing evaluation strategies -- when to use which method, how to size test suites, and how to integrate evals into CI/CD",
"tags": [
"evaluation",
"testing",
"llm-as-judge",
"regression",
"confidence-intervals",
"ci-cd"
],
"phase": 11,
"lesson": 10,
"lessonPath": "phases/11-llm-engineering/10-evaluation",
"file": "phases/11-llm-engineering/10-evaluation/outputs/skill-eval-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-cost-optimizer",
"description": "Analyze an LLM application and recommend specific cost optimizations with projected savings",
"tags": [],
"phase": 11,
"lesson": 11,
"lessonPath": "phases/11-llm-engineering/11-caching-cost",
"file": "phases/11-llm-engineering/11-caching-cost/outputs/prompt-cost-optimizer.md"
},
{
"kind": "skill",
"name": "skill-cost-patterns",
"description": "Decision framework for LLM cost optimization -- caching strategies, rate limiting, model routing, and budget controls",
"tags": [
"caching",
"cost-optimization",
"rate-limiting",
"model-routing",
"budget",
"llm-ops"
],
"phase": 11,
"lesson": 11,
"lessonPath": "phases/11-llm-engineering/11-caching-cost",
"file": "phases/11-llm-engineering/11-caching-cost/outputs/skill-cost-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-safety-auditor",
"description": "Audit any LLM application for safety vulnerabilities -- prompt injection, data leakage, jailbreaks, and output risks",
"tags": [],
"phase": 11,
"lesson": 12,
"lessonPath": "phases/11-llm-engineering/12-guardrails",
"file": "phases/11-llm-engineering/12-guardrails/outputs/prompt-safety-auditor.md"
},
{
"kind": "skill",
"name": "skill-guardrail-patterns",
"description": "Decision framework for choosing and implementing guardrails in production -- tool selection, layering strategy, and cost-performance tradeoffs",
"tags": [
"guardrails",
"safety",
"content-filtering",
"prompt-injection",
"pii",
"moderation",
"llamaguard",
"nemo"
],
"phase": 11,
"lesson": 12,
"lessonPath": "phases/11-llm-engineering/12-guardrails",
"file": "phases/11-llm-engineering/12-guardrails/outputs/skill-guardrail-patterns.md"
},
{
"kind": "prompt",
"name": "prompt-architecture-reviewer",
"description": "Review the architecture of any LLM application against a production readiness checklist -- identifies gaps, risks, and missing components",
"tags": [],
"phase": 11,
"lesson": 13,
"lessonPath": "phases/11-llm-engineering/13-production-app",
"file": "phases/11-llm-engineering/13-production-app/outputs/prompt-architecture-reviewer.md"
},
{
"kind": "skill",
"name": "skill-production-checklist",
"description": "Decision framework for shipping LLM applications to production -- covers every component with specific thresholds and pass/fail criteria",
"tags": [
"production",
"deployment",
"llm",
"architecture",
"scaling",
"cost",
"observability",
"guardrails"
],
"phase": 11,
"lesson": 13,
"lessonPath": "phases/11-llm-engineering/13-production-app",
"file": "phases/11-llm-engineering/13-production-app/outputs/skill-production-checklist.md"
},
{
"kind": "skill",
"name": "mcp-server-designer",
"description": "Design and scaffold an MCP server with tools, resources, and safety defaults.",
"tags": [
"llm-engineering",
"mcp",
"tool-use"
],
"phase": 11,
"lesson": 14,
"lessonPath": "phases/11-llm-engineering/14-model-context-protocol",
"file": "phases/11-llm-engineering/14-model-context-protocol/outputs/skill-mcp-server-designer.md"
},
{
"kind": "skill",
"name": "prompt-caching-planner",
"description": "Design a cache-friendly prompt layout and pick the right provider caching mode.",
"tags": [
"llm-engineering",
"caching",
"cost"
],
"phase": 11,
"lesson": 15,
"lessonPath": "phases/11-llm-engineering/15-prompt-caching",
"file": "phases/11-llm-engineering/15-prompt-caching/outputs/skill-prompt-caching-planner.md"
},
{
"kind": "skill",
"name": "stategraph-designer",
"description": "Turn an agent task into a LangGraph StateGraph with named nodes, typed state, reducers, checkpointer, and human interrupts.",
"tags": [
"langgraph",
"stategraph",
"checkpointer",
"interrupt",
"time-travel",
"react-agent",
"human-in-the-loop"
],
"phase": 11,
"lesson": 16,
"lessonPath": "phases/11-llm-engineering/16-langgraph-state-machines",
"file": "phases/11-llm-engineering/16-langgraph-state-machines/outputs/skill-stategraph-designer.md"
},
{
"kind": "skill",
"name": "framework-picker",
"description": "Pick LangGraph, CrewAI, AutoGen, Agno, or plain Python for an agent task by matching abstraction to problem shape.",
"tags": [
"langgraph",
"crewai",
"autogen",
"agno",
"agent-framework",
"orchestration",
"decision-matrix"
],
"phase": 11,
"lesson": 17,
"lessonPath": "phases/11-llm-engineering/17-agent-framework-tradeoffs",
"file": "phases/11-llm-engineering/17-agent-framework-tradeoffs/outputs/skill-framework-picker.md"
},
{
"kind": "skill",
"name": "patch-geometry-reader",
"description": "Read a ViT config and produce a patch-token, parameter, and VRAM analysis for downstream VLM planning.",
"tags": [
"vit",
"patch-tokens",
"dinov2",
"siglip",
"vlm-backbone"
],
"phase": 12,
"lesson": 1,
"lessonPath": "phases/12-multimodal-ai/01-vision-transformer-patch-tokens",
"file": "phases/12-multimodal-ai/01-vision-transformer-patch-tokens/outputs/skill-patch-geometry-reader.md"
},
{
"kind": "skill",
"name": "clip-zero-shot",
"description": "Run zero-shot image classification with a CLIP / SigLIP checkpoint, producing ranked predictions with similarity scores.",
"tags": [
"clip",
"siglip",
"zero-shot",
"vision-language"
],
"phase": 12,
"lesson": 2,
"lessonPath": "phases/12-multimodal-ai/02-clip-contrastive-pretraining",
"file": "phases/12-multimodal-ai/02-clip-contrastive-pretraining/outputs/skill-clip-zero-shot.md"
},
{
"kind": "skill",
"name": "modality-bridge-picker",
"description": "Recommend Q-Former vs MLP projector vs Perceiver resampler for a VLM configuration given token budget, quality target, and training compute.",
"tags": [
"blip2",
"qformer",
"vlm",
"modality-bridge",
"architecture"
],
"phase": 12,
"lesson": 3,
"lessonPath": "phases/12-multimodal-ai/03-blip2-qformer-bridge",
"file": "phases/12-multimodal-ai/03-blip2-qformer-bridge/outputs/skill-modality-bridge-picker.md"
},
{
"kind": "skill",
"name": "gated-bridge-diagnostic",
"description": "Identify Flamingo-lineage design elements in an open VLM config and diagnose freezing / gating issues.",
"tags": [
"flamingo",
"idefics",
"openflamingo",
"gated-cross-attention",
"interleaved-inputs"
],
"phase": 12,
"lesson": 4,
"lessonPath": "phases/12-multimodal-ai/04-flamingo-gated-cross-attention",
"file": "phases/12-multimodal-ai/04-flamingo-gated-cross-attention/outputs/skill-gated-bridge-diagnostic.md"
},
{
"kind": "skill",
"name": "llava-vibes-eval",
"description": "Run a 10-prompt vibes-eval on a LLaVA-family VLM and produce a human-readable scorecard.",
"tags": [
"llava",
"vlm",
"vibes-eval",
"instruction-tuning"
],
"phase": 12,
"lesson": 5,
"lessonPath": "phases/12-multimodal-ai/05-llava-visual-instruction-tuning",
"file": "phases/12-multimodal-ai/05-llava-visual-instruction-tuning/outputs/skill-llava-vibes-eval.md"
},
{
"kind": "skill",
"name": "resolution-budget-planner",
"description": "Pick between square-resize, AnyRes, M-RoPE, and NaFlex for a mixed-aspect-ratio VLM workload and emit a per-task token budget plan.",
"tags": [
"vlm",
"patch-n-pack",
"naflex",
"anyres",
"m-rope",
"token-budget"
],
"phase": 12,
"lesson": 6,
"lessonPath": "phases/12-multimodal-ai/06-any-resolution-patch-n-pack",
"file": "phases/12-multimodal-ai/06-any-resolution-patch-n-pack/outputs/skill-resolution-budget-planner.md"
},
{
"kind": "skill",
"name": "vlm-recipe-picker",
"description": "Pick an open-weight VLM recipe (encoder, connector, LLM, data mix, resolution schedule) with ablation-table citations for every choice.",
"tags": [
"vlm",
"mm1",
"idefics2",
"molmo",
"cambrian",
"prismatic",
"ablation"
],
"phase": 12,
"lesson": 7,
"lessonPath": "phases/12-multimodal-ai/07-open-weight-vlm-recipes",
"file": "phases/12-multimodal-ai/07-open-weight-vlm-recipes/outputs/skill-vlm-recipe-picker.md"
},
{
"kind": "skill",
"name": "onevision-budget-planner",
"description": "Allocate LLaVA-OneVision-style unified visual-token budgets across single-image, multi-image, and video scenarios for a target product mix.",
"tags": [
"llava-onevision",
"token-budget",
"curriculum",
"multi-image",
"video"
],
"phase": 12,
"lesson": 8,
"lessonPath": "phases/12-multimodal-ai/08-llava-onevision-single-multi-video",
"file": "phases/12-multimodal-ai/08-llava-onevision-single-multi-video/outputs/skill-onevision-budget-planner.md"
},
{
"kind": "skill",
"name": "qwen-vl-pipeline-designer",
"description": "Configure a Qwen2.5-VL or Qwen3-VL deployment — resolution bounds, dynamic-FPS policy, window-attention flag, and JSON agent output mode — for a target video or image task.",
"tags": [
"qwen-vl",
"m-rope",
"dynamic-fps",
"json-agent",
"video-understanding"
],
"phase": 12,
"lesson": 9,
"lessonPath": "phases/12-multimodal-ai/09-qwen-vl-family-dynamic-fps",
"file": "phases/12-multimodal-ai/09-qwen-vl-family-dynamic-fps/outputs/skill-qwen-vl-pipeline-designer.md"
},
{
"kind": "skill",
"name": "native-vs-posthoc-auditor",
"description": "Audit a proposed VLM training plan and recommend native multimodal pretraining or post-hoc adapter-on-LLM, with corpus-mix and alignment-debt analysis.",
"tags": [
"internvl3",
"native-pretraining",
"post-hoc",
"corpus-mix",
"alignment-debt"
],
"phase": 12,
"lesson": 10,
"lessonPath": "phases/12-multimodal-ai/10-internvl3-native-multimodal",
"file": "phases/12-multimodal-ai/10-internvl3-native-multimodal/outputs/skill-native-vs-posthoc-auditor.md"
},
{
"kind": "skill",
"name": "tokenizer-vs-adapter-picker",
"description": "Pick between Chameleon-style early fusion (shared-vocab tokenizer) and LLaVA-style late fusion (adapter on frozen LLM) for a VLM project.",
"tags": [
"chameleon",
"early-fusion",
"vq-vae",
"late-fusion",
"adapter"
],
"phase": 12,
"lesson": 11,
"lessonPath": "phases/12-multimodal-ai/11-chameleon-early-fusion-tokens",
"file": "phases/12-multimodal-ai/11-chameleon-early-fusion-tokens/outputs/skill-tokenizer-vs-adapter-picker.md"
},
{
"kind": "skill",
"name": "token-gen-cost-analyzer",
"description": "Compute token counts, inference latency, and quality ceiling for Emu3-style next-token generation and pick between Emu3-family and diffusion.",
"tags": [
"emu3",
"next-token-prediction",
"video-gen",
"diffusion",
"cfg"
],
"phase": 12,
"lesson": 12,
"lessonPath": "phases/12-multimodal-ai/12-emu3-next-token-for-generation",
"file": "phases/12-multimodal-ai/12-emu3-next-token-for-generation/outputs/skill-token-gen-cost-analyzer.md"
},
{
"kind": "skill",
"name": "two-loss-trainer-designer",
"description": "Design a Transfusion / MMDiT-style two-loss training setup (NTP on one modality, diffusion on another) with loss weights, mask design, and schedule.",
"tags": [
"transfusion",
"mmdit",
"two-loss",
"flow-matching",
"hybrid-attention"
],
"phase": 12,
"lesson": 13,
"lessonPath": "phases/12-multimodal-ai/13-transfusion-autoregressive-diffusion",
"file": "phases/12-multimodal-ai/13-transfusion-autoregressive-diffusion/outputs/skill-two-loss-trainer-designer.md"
},
{
"kind": "skill",
"name": "unified-gen-model-picker",
"description": "Pick between Show-o / Transfusion / Emu3 / Janus-Pro families for a product that needs both multimodal understanding and generation with open weights.",
"tags": [
"show-o",
"masked-diffusion",
"unified",
"t2i",
"inpainting"
],
"phase": 12,
"lesson": 14,
"lessonPath": "phases/12-multimodal-ai/14-show-o-discrete-diffusion-unified",
"file": "phases/12-multimodal-ai/14-show-o-discrete-diffusion-unified/outputs/skill-unified-gen-model-picker.md"
},
{
"kind": "skill",
"name": "decoupled-encoder-picker",
"description": "Decide whether a unified VLM should decouple its visual encoders and pick between Janus-Pro, JanusFlow, and InternVL-U.",
"tags": [
"janus-pro",
"janusflow",
"internvl-u",
"decoupled-encoders",
"unified-model"
],
"phase": 12,
"lesson": 15,
"lessonPath": "phases/12-multimodal-ai/15-janus-pro-decoupled-encoders",
"file": "phases/12-multimodal-ai/15-janus-pro-decoupled-encoders/outputs/skill-decoupled-encoder-picker.md"
},
{
"kind": "skill",
"name": "any-to-any-pipeline-auditor",
"description": "Audit a conversational any-to-any design and compute the latency budget for a MIO / AnyGPT / Moshi-family stack.",
"tags": [
"mio",
"anygpt",
"moshi",
"any-to-any",
"streaming",
"ttfab"
],
"phase": 12,
"lesson": 16,
"lessonPath": "phases/12-multimodal-ai/16-mio-any-to-any-streaming",
"file": "phases/12-multimodal-ai/16-mio-any-to-any-streaming/outputs/skill-any-to-any-pipeline-auditor.md"
},
{
"kind": "skill",
"name": "video-vlm-frame-planner",
"description": "Plan frame sampling, per-frame pooling, output format, and benchmark targets for a video-language model deployment.",
"tags": [
"video-vlm",
"temporal-grounding",
"tmrope",
"dynamic-fps",
"benchmarks"
],
"phase": 12,
"lesson": 17,
"lessonPath": "phases/12-multimodal-ai/17-video-language-temporal-grounding",
"file": "phases/12-multimodal-ai/17-video-language-temporal-grounding/outputs/skill-video-vlm-frame-planner.md"
},
{
"kind": "skill",
"name": "long-video-strategy-planner",
"description": "Pick brute-context, ring-attention, token-compression, or agentic-retrieval for a long-video understanding task and compute latency + recall expectations.",
"tags": [
"long-video",
"gemini",
"ring-attention",
"videoagent",
"retrieval"
],
"phase": 12,
"lesson": 18,
"lessonPath": "phases/12-multimodal-ai/18-long-video-million-token",
"file": "phases/12-multimodal-ai/18-long-video-million-token/outputs/skill-long-video-strategy-planner.md"
},
{
"kind": "skill",
"name": "audio-llm-pipeline-picker",
"description": "Pick cascaded (Whisper + LLM) or end-to-end (AF3 / Qwen-Audio) for an audio task, plus the encoder and bridge config.",
"tags": [
"whisper",
"audio-flamingo-3",
"qwen-audio",
"cascaded",
"end-to-end"
],
"phase": 12,
"lesson": 19,
"lessonPath": "phases/12-multimodal-ai/19-audio-language-whisper-to-af3",
"file": "phases/12-multimodal-ai/19-audio-language-whisper-to-af3/outputs/skill-audio-llm-pipeline-picker.md"
},
{
"kind": "skill",
"name": "omni-streaming-budget",
"description": "Size a Thinker-Talker streaming voice pipeline (Qwen-Omni / Moshi / Mini-Omni) for a target TTFAB and feature set.",
"tags": [
"qwen-omni",
"moshi",
"mini-omni",
"streaming",
"ttfab",
"thinker-talker"
],
"phase": 12,
"lesson": 20,
"lessonPath": "phases/12-multimodal-ai/20-omni-models-thinker-talker",
"file": "phases/12-multimodal-ai/20-omni-models-thinker-talker/outputs/skill-omni-streaming-budget.md"
},
{
"kind": "skill",
"name": "vla-action-format-picker",
"description": "Pick an action format (discrete bin, FAST, flow-matching, dual-system) and VLA family (RT-2, OpenVLA, π0, GR00T) for a robot task.",
"tags": [
"vla",
"rt-2",
"openvla",
"pi0",
"groot",
"action-tokenization"
],
"phase": 12,
"lesson": 21,
"lessonPath": "phases/12-multimodal-ai/21-embodied-vlas-openvla-pi0-groot",
"file": "phases/12-multimodal-ai/21-embodied-vlas-openvla-pi0-groot/outputs/skill-vla-action-format-picker.md"
},
{
"kind": "skill",
"name": "document-ai-stack-picker",
"description": "Pick between OCR pipeline, OCR-free specialist, and VLM-native for a document-AI project based on domain, scale, and regulatory needs.",
"tags": [
"document-ai",
"ocr",
"donut",
"nougat",
"paligemma",
"vlm-native"
],
"phase": 12,
"lesson": 22,
"lessonPath": "phases/12-multimodal-ai/22-document-diagram-understanding",
"file": "phases/12-multimodal-ai/22-document-diagram-understanding/outputs/skill-document-ai-stack-picker.md"
},
{
"kind": "skill",
"name": "vision-rag-designer",
"description": "Design a vision-native document RAG using ColPali / ColQwen2 / VisRAG, with storage estimate and generator-pick.",
"tags": [
"colpali",
"colqwen2",
"visrag",
"late-interaction",
"vidore"
],
"phase": 12,
"lesson": 23,
"lessonPath": "phases/12-multimodal-ai/23-colpali-vision-native-rag",
"file": "phases/12-multimodal-ai/23-colpali-vision-native-rag/outputs/skill-vision-rag-designer.md"
},
{
"kind": "skill",
"name": "multimodal-rag-designer",
"description": "Design a production multimodal RAG across text, images, audio, video with retrievers, fusion strategy, and grounded generator.",
"tags": [
"multimodal-rag",
"cross-modal-retrieval",
"fusion",
"grounded-generation"
],
"phase": 12,
"lesson": 24,
"lessonPath": "phases/12-multimodal-ai/24-multimodal-rag-cross-modal",
"file": "phases/12-multimodal-ai/24-multimodal-rag-cross-modal/outputs/skill-multimodal-rag-designer.md"
},
{
"kind": "skill",
"name": "multimodal-agent-designer",
"description": "Design a multimodal agent (computer-use, GUI grounding, web or mobile) with action schema, memory strategy, and benchmark evaluation plan.",
"tags": [
"multimodal-agents",
"computer-use",
"gui-grounding",
"visualwebarena",
"agentvista"
],
"phase": 12,
"lesson": 25,
"lessonPath": "phases/12-multimodal-ai/25-multimodal-agents-computer-use",
"file": "phases/12-multimodal-ai/25-multimodal-agents-computer-use/outputs/skill-multimodal-agent-designer.md"
},
{
"kind": "skill",
"name": "tool-interface-reviewer",
"description": "Audit a tool definition (name + description + JSON Schema + executor outline) for loop fitness before it ships to an LLM.",
"tags": [
"tool-calling",
"function-calling",
"json-schema",
"tool-design"
],
"phase": 13,
"lesson": 1,
"lessonPath": "phases/13-tools-and-protocols/01-the-tool-interface",
"file": "phases/13-tools-and-protocols/01-the-tool-interface/outputs/skill-tool-interface-reviewer.md"
},
{
"kind": "skill",
"name": "provider-portability-audit",
"description": "Audit a function-calling integration against one provider for what breaks when ported to the other two.",
"tags": [
"function-calling",
"openai",
"anthropic",
"gemini",
"portability"
],
"phase": 13,
"lesson": 2,
"lessonPath": "phases/13-tools-and-protocols/02-function-calling-deep-dive",
"file": "phases/13-tools-and-protocols/02-function-calling-deep-dive/outputs/skill-provider-portability-audit.md"
},
{
"kind": "skill",
"name": "parallel-call-safety-check",
"description": "Audit a tool registry for safe parallelization. Mark each tool parallel_safe, note ordering dependencies, and flag downstream rate-limit risk.",
"tags": [
"parallel-tool-calls",
"streaming",
"correlation",
"rate-limits"
],
"phase": 13,
"lesson": 3,
"lessonPath": "phases/13-tools-and-protocols/03-parallel-and-streaming-tool-calls",
"file": "phases/13-tools-and-protocols/03-parallel-and-streaming-tool-calls/outputs/skill-parallel-call-safety-check.md"
},
{
"kind": "skill",
"name": "structured-output-designer",
"description": "Design a strict-mode-compatible JSON Schema plus Pydantic model for a free-text extraction target, with typed refusal and retry handling stubbed in.",
"tags": [
"structured-output",
"json-schema",
"pydantic",
"strict-mode",
"extraction"
],
"phase": 13,
"lesson": 4,
"lessonPath": "phases/13-tools-and-protocols/04-structured-output",
"file": "phases/13-tools-and-protocols/04-structured-output/outputs/skill-structured-output-designer.md"
},
{
"kind": "skill",
"name": "tool-schema-linter",
"description": "Audit a tool registry against production design rules for names, descriptions, parameters, and shape. Can run in CI on every tool-registry change.",
"tags": [
"tool-design",
"linter",
"selection-accuracy",
"naming"
],
"phase": 13,
"lesson": 5,
"lessonPath": "phases/13-tools-and-protocols/05-tool-schema-design",
"file": "phases/13-tools-and-protocols/05-tool-schema-design/outputs/skill-tool-schema-linter.md"
},
{
"kind": "skill",
"name": "mcp-handshake-tracer",
"description": "Given a pcap-style transcript of an MCP client-server conversation, annotate every message with its primitive, lifecycle phase, and capability dependency.",
"tags": [
"mcp",
"json-rpc",
"lifecycle",
"capabilities"
],
"phase": 13,
"lesson": 6,
"lessonPath": "phases/13-tools-and-protocols/06-mcp-fundamentals",
"file": "phases/13-tools-and-protocols/06-mcp-fundamentals/outputs/skill-mcp-handshake-tracer.md"
},
{
"kind": "skill",
"name": "mcp-server-scaffolder",
"description": "Scaffold a domain-specific MCP server with the right tools/resources/prompts split and SDK graduation path.",
"tags": [
"mcp",
"server",
"fastmcp",
"scaffold"
],
"phase": 13,
"lesson": 7,
"lessonPath": "phases/13-tools-and-protocols/07-building-an-mcp-server",
"file": "phases/13-tools-and-protocols/07-building-an-mcp-server/outputs/skill-mcp-server-scaffolder.md"
},
{
"kind": "skill",
"name": "mcp-client-harness",
"description": "Given a declarative list of MCP servers (name, command, args), scaffold a multi-server client with handshake, namespace merge, and routing.",
"tags": [
"mcp",
"client",
"multi-server",
"routing",
"namespace"
],
"phase": 13,
"lesson": 8,
"lessonPath": "phases/13-tools-and-protocols/08-building-an-mcp-client",
"file": "phases/13-tools-and-protocols/08-building-an-mcp-client/outputs/skill-mcp-client-harness.md"
},
{
"kind": "skill",
"name": "mcp-transport-migrator",
"description": "Produce a migration plan from legacy HTTP+SSE to Streamable HTTP with session id continuity and Origin validation.",
"tags": [
"mcp",
"streamable-http",
"sse-migration",
"session-id",
"origin"
],
"phase": 13,
"lesson": 9,
"lessonPath": "phases/13-tools-and-protocols/09-mcp-transports",
"file": "phases/13-tools-and-protocols/09-mcp-transports/outputs/skill-mcp-transport-migrator.md"
},
{
"kind": "skill",
"name": "primitive-splitter",
"description": "Categorize each capability in an MCP server draft as tool, resource, or prompt with rationale.",
"tags": [
"mcp",
"primitives",
"resources",
"prompts"
],
"phase": 13,
"lesson": 10,
"lessonPath": "phases/13-tools-and-protocols/10-mcp-resources-and-prompts",
"file": "phases/13-tools-and-protocols/10-mcp-resources-and-prompts/outputs/skill-primitive-splitter.md"
},
{
"kind": "skill",
"name": "sampling-loop-designer",
"description": "Design a server-hosted agent loop using MCP sampling with the right modelPreferences, rate limits, and safety confirmations.",
"tags": [
"mcp",
"sampling",
"agent-loop",
"model-preferences"
],
"phase": 13,
"lesson": 11,
"lessonPath": "phases/13-tools-and-protocols/11-mcp-sampling",
"file": "phases/13-tools-and-protocols/11-mcp-sampling/outputs/skill-sampling-loop-designer.md"
},
{
"kind": "skill",
"name": "elicitation-form-designer",
"description": "Design the elicitation form schema and message template for a tool that needs mid-call user confirmation or disambiguation.",
"tags": [
"mcp",
"elicitation",
"user-input",
"forms"
],
"phase": 13,
"lesson": 12,
"lessonPath": "phases/13-tools-and-protocols/12-mcp-roots-and-elicitation",
"file": "phases/13-tools-and-protocols/12-mcp-roots-and-elicitation/outputs/skill-elicitation-form-designer.md"
},
{
"kind": "skill",
"name": "task-store-designer",
"description": "Design the task store for a long-running MCP tool: state shape, ttl, durability, cancellation, crash recovery.",
"tags": [
"mcp",
"tasks",
"durable-store",
"long-running",
"sep-1686"
],
"phase": 13,
"lesson": 13,
"lessonPath": "phases/13-tools-and-protocols/13-mcp-async-tasks",
"file": "phases/13-tools-and-protocols/13-mcp-async-tasks/outputs/skill-task-store-designer.md"
},
{
"kind": "skill",
"name": "mcp-apps-spec",
"description": "Produce the full MCP Apps contract for a tool that needs an interactive UI resource.",
"tags": [
"mcp",
"apps",
"ui-resources",
"csp",
"iframe-sandbox"
],
"phase": 13,
"lesson": 14,
"lessonPath": "phases/13-tools-and-protocols/14-mcp-apps",
"file": "phases/13-tools-and-protocols/14-mcp-apps/outputs/skill-mcp-apps-spec.md"
},
{
"kind": "skill",
"name": "mcp-threat-model",
"description": "Produce a threat model for an MCP deployment naming the applicable attack classes, defenses in place, and Rule-of-Two violations.",
"tags": [
"mcp",
"security",
"tool-poisoning",
"threat-model",
"rule-of-two"
],
"phase": 13,
"lesson": 15,
"lessonPath": "phases/13-tools-and-protocols/15-mcp-security-tool-poisoning",
"file": "phases/13-tools-and-protocols/15-mcp-security-tool-poisoning/outputs/skill-mcp-threat-model.md"
},
{
"kind": "skill",
"name": "oauth-scope-planner",
"description": "Design the OAuth 2.1 scope set, pinning rules, and step-up policy for a remote MCP server.",
"tags": [
"oauth",
"pkce",
"resource-indicators",
"step-up",
"sep-835"
],
"phase": 13,
"lesson": 16,
"lessonPath": "phases/13-tools-and-protocols/16-mcp-security-oauth-2-1",
"file": "phases/13-tools-and-protocols/16-mcp-security-oauth-2-1/outputs/skill-oauth-scope-planner.md"
},
{
"kind": "skill",
"name": "gateway-bootstrap",
"description": "Produce a gateway configuration spec given users, backends, and compliance constraints.",
"tags": [
"mcp",
"gateway",
"rbac",
"audit",
"policy"
],
"phase": 13,
"lesson": 17,
"lessonPath": "phases/13-tools-and-protocols/17-mcp-gateways-and-registries",
"file": "phases/13-tools-and-protocols/17-mcp-gateways-and-registries/outputs/skill-gateway-bootstrap.md"
},
{
"kind": "skill",
"name": "mcp-auth-wiring",
"description": "Stand up production MCP authorization (RFC 8414, CIMD, 7591, 8707, 7636 PKCE, 9728, 9207) — protected-resource metadata, enrollment, JWKS refresh, and per-request token validation.",
"tags": [
"mcp",
"oauth",
"cimd",
"dcr",
"jwks",
"rfc8414",
"rfc7591",
"rfc8707",
"rfc7636",
"rfc9728",
"rfc9207"
],
"phase": 13,
"lesson": 18,
"lessonPath": "phases/13-tools-and-protocols/18-mcp-auth-production",
"file": "phases/13-tools-and-protocols/18-mcp-auth-production/outputs/skill-mcp-auth.md"
},
{
"kind": "skill",
"name": "a2a-agent-spec",
"description": "Produce the Agent Card and skills schema for an agent that should be callable over A2A.",
"tags": [
"a2a",
"agent-card",
"task-lifecycle",
"delegation"
],
"phase": 13,
"lesson": 19,
"lessonPath": "phases/13-tools-and-protocols/19-a2a-protocol",
"file": "phases/13-tools-and-protocols/19-a2a-protocol/outputs/skill-a2a-agent-spec.md"
},
{
"kind": "skill",
"name": "otel-genai-instrumentation",
"description": "Produce an instrumentation plan for an agent codebase to emit OTel GenAI spans end-to-end.",
"tags": [
"otel",
"observability",
"gen-ai",
"tracing"
],
"phase": 13,
"lesson": 20,
"lessonPath": "phases/13-tools-and-protocols/20-opentelemetry-genai",
"file": "phases/13-tools-and-protocols/20-opentelemetry-genai/outputs/skill-otel-genai-instrumentation.md"
},
{
"kind": "skill",
"name": "routing-config-designer",
"description": "Given a workload profile, pick LiteLLM / OpenRouter / Portkey and produce a routing config.",
"tags": [
"routing",
"litellm",
"openrouter",
"portkey",
"fallback"
],
"phase": 13,
"lesson": 21,
"lessonPath": "phases/13-tools-and-protocols/21-llm-routing-layer",
"file": "phases/13-tools-and-protocols/21-llm-routing-layer/outputs/skill-routing-config-designer.md"
},
{
"kind": "skill",
"name": "agent-bundle",
"description": "Produce a portable SKILL.md + AGENTS.md + MCP-server blueprint for a workflow, loadable across Claude Code, Cursor, Codex, and compatible agents.",
"tags": [
"skills",
"agents-md",
"apps-sdk",
"cross-agent",
"portability"
],
"phase": 13,
"lesson": 22,
"lessonPath": "phases/13-tools-and-protocols/22-skills-and-agent-sdks",
"file": "phases/13-tools-and-protocols/22-skills-and-agent-sdks/outputs/skill-agent-bundle.md"
},
{
"kind": "skill",
"name": "ecosystem-blueprint",
"description": "Produce a full Phase 13 ecosystem architecture given a product need; name primitives, security posture, telemetry, and packaging.",
"tags": [
"mcp",
"capstone",
"ecosystem",
"architecture",
"a2a",
"otel"
],
"phase": 13,
"lesson": 23,
"lessonPath": "phases/13-tools-and-protocols/23-capstone-tool-ecosystem",
"file": "phases/13-tools-and-protocols/23-capstone-tool-ecosystem/outputs/skill-ecosystem-blueprint.md"
},
{
"kind": "skill",
"name": "agent-loop",
"description": "Write a correct, minimal ReAct agent loop in any target language/runtime with tools, stop condition, and turn budget.",
"tags": [
"react",
"agent-loop",
"tools",
"observability",
"stop-condition"
],
"phase": 14,
"lesson": 1,
"lessonPath": "phases/14-agent-engineering/01-the-agent-loop",
"file": "phases/14-agent-engineering/01-the-agent-loop/outputs/skill-agent-loop.md"
},
{
"kind": "skill",
"name": "rewoo-planner",
"description": "Generate a validated ReWOO plan DAG from a user request and tool catalog.",
"tags": [
"rewoo",
"plan-and-execute",
"planning",
"dag",
"distillation"
],
"phase": 14,
"lesson": 2,
"lessonPath": "phases/14-agent-engineering/02-rewoo-plan-and-execute",
"file": "phases/14-agent-engineering/02-rewoo-plan-and-execute/outputs/skill-rewoo-planner.md"
},
{
"kind": "skill",
"name": "reflexion-buffer",
"description": "Maintain an episodic-memory buffer of reflections for verbal RL with TTL, dedup, and scoped scope.",
"tags": [
"reflexion",
"episodic-memory",
"self-healing",
"verbal-rl",
"sleep-time"
],
"phase": 14,
"lesson": 3,
"lessonPath": "phases/14-agent-engineering/03-reflexion-verbal-rl",
"file": "phases/14-agent-engineering/03-reflexion-verbal-rl/outputs/skill-reflexion-buffer.md"
},
{
"kind": "skill",
"name": "search-policy",
"description": "Pick a search strategy (ReAct, ToT, LATS, evolutionary) given task shape, token budget, and evaluator quality.",
"tags": [
"tree-of-thoughts",
"lats",
"mcts",
"search",
"value-function"
],
"phase": 14,
"lesson": 4,
"lessonPath": "phases/14-agent-engineering/04-tree-of-thoughts-lats",
"file": "phases/14-agent-engineering/04-tree-of-thoughts-lats/outputs/skill-search-policy.md"
},
{
"kind": "skill",
"name": "refine-loop",
"description": "Configure an evaluator-optimizer (Self-Refine / CRITIC) loop given task, verifier availability, and iteration budget.",
"tags": [
"self-refine",
"critic",
"evaluator-optimizer",
"guardrails",
"iteration"
],
"phase": 14,
"lesson": 5,
"lessonPath": "phases/14-agent-engineering/05-self-refine-and-critic",
"file": "phases/14-agent-engineering/05-self-refine-and-critic/outputs/skill-refine-loop.md"
},
{
"kind": "skill",
"name": "tool-registry",
"description": "Build a production tool catalog and registry with JSON Schema validation, parallel dispatch, and observability.",
"tags": [
"function-calling",
"tools",
"schema",
"validation",
"bfcl",
"parallel-tools"
],
"phase": 14,
"lesson": 6,
"lessonPath": "phases/14-agent-engineering/06-tool-use-and-function-calling",
"file": "phases/14-agent-engineering/06-tool-use-and-function-calling/outputs/skill-tool-registry.md"
},
{
"kind": "skill",
"name": "virtual-memory",
"description": "Scaffold a MemGPT-shaped two-tier memory system (main context + archival store + memory tools) for any target runtime with correct eviction, citation, and untrusted-input handling.",
"tags": [
"memory",
"memgpt",
"virtual-context",
"archival",
"citations"
],
"phase": 14,
"lesson": 7,
"lessonPath": "phases/14-agent-engineering/07-memory-virtual-context-memgpt",
"file": "phases/14-agent-engineering/07-memory-virtual-context-memgpt/outputs/skill-virtual-memory.md"
},
{
"kind": "skill",
"name": "memory-blocks",
"description": "Generate a Letta-shaped three-tier memory system (core blocks, recall, archival) with a sleep-time consolidation agent off the critical path.",
"tags": [
"memory",
"letta",
"blocks",
"sleep-time",
"consolidation"
],
"phase": 14,
"lesson": 8,
"lessonPath": "phases/14-agent-engineering/08-memory-blocks-sleep-time-compute",
"file": "phases/14-agent-engineering/08-memory-blocks-sleep-time-compute/outputs/skill-memory-blocks.md"
},
{
"kind": "skill",
"name": "hybrid-memory",
"description": "Generate a Mem0-shaped three-store memory system (vector + KV + graph) with a fusion scorer, scope taxonomy, and temporal invalidation.",
"tags": [
"memory",
"mem0",
"vector",
"graph",
"kv",
"fusion",
"scope"
],
"phase": 14,
"lesson": 9,
"lessonPath": "phases/14-agent-engineering/09-hybrid-memory-mem0",
"file": "phases/14-agent-engineering/09-hybrid-memory-mem0/outputs/skill-hybrid-memory.md"
},
{
"kind": "skill",
"name": "skill-library",
"description": "Generate a Voyager-shaped skill library with registration, retrieval by similarity, compositional execution, and failure-driven refinement.",
"tags": [
"voyager",
"skills",
"library",
"composition",
"refinement"
],
"phase": 14,
"lesson": 10,
"lessonPath": "phases/14-agent-engineering/10-skill-libraries-voyager",
"file": "phases/14-agent-engineering/10-skill-libraries-voyager/outputs/skill-skill-library.md"
},
{
"kind": "skill",
"name": "hybrid-planner",
"description": "Build a hybrid planner — ChatHTN for provably-sound plans, AlphaEvolve for code search with a machine-checkable evaluator — and pick the right one for the problem.",
"tags": [
"planning",
"htn",
"chathtn",
"alphaevolve",
"evolutionary-search"
],
"phase": 14,
"lesson": 11,
"lessonPath": "phases/14-agent-engineering/11-planning-htn-and-evolutionary",
"file": "phases/14-agent-engineering/11-planning-htn-and-evolutionary/outputs/skill-hybrid-planner.md"
},
{
"kind": "skill",
"name": "workflow-picker",
"description": "Pick the right pattern (prompt chain, router, parallel, orchestrator-workers, evaluator-optimizer, or full agent) for a given task and produce the minimal implementation.",
"tags": [
"anthropic",
"workflows",
"agents",
"patterns",
"minimal"
],
"phase": 14,
"lesson": 12,
"lessonPath": "phases/14-agent-engineering/12-anthropic-workflow-patterns",
"file": "phases/14-agent-engineering/12-anthropic-workflow-patterns/outputs/skill-workflow-picker.md"
},
{
"kind": "skill",
"name": "state-graph",
"description": "Build a LangGraph-shaped state machine with typed state, conditional edges, per-node checkpointing, and durable resume.",
"tags": [
"langgraph",
"state-machine",
"durable",
"checkpointing",
"human-in-the-loop"
],
"phase": 14,
"lesson": 13,
"lessonPath": "phases/14-agent-engineering/13-langgraph-stateful-graphs",
"file": "phases/14-agent-engineering/13-langgraph-stateful-graphs/outputs/skill-state-graph.md"
},
{
"kind": "skill",
"name": "actor-runtime",
"description": "Build an AutoGen v0.4-shaped actor runtime with private state, inbox-per-actor, message-only IPC, fault isolation, and a dead-letter queue.",
"tags": [
"autogen",
"actor-model",
"messaging",
"fault-isolation",
"dead-letter"
],
"phase": 14,
"lesson": 14,
"lessonPath": "phases/14-agent-engineering/14-autogen-actor-model",
"file": "phases/14-agent-engineering/14-autogen-actor-model/outputs/skill-actor-runtime.md"
},
{
"kind": "skill",
"name": "crew-or-flow",
"description": "Pick CrewAI Crew or Flow for a given task, and scaffold the minimal implementation.",
"tags": [
"crewai",
"crews",
"flows",
"multi-agent",
"role-based"
],
"phase": 14,
"lesson": 15,
"lessonPath": "phases/14-agent-engineering/15-crewai-role-based-crews",
"file": "phases/14-agent-engineering/15-crewai-role-based-crews/outputs/skill-crew-or-flow.md"
},
{
"kind": "skill",
"name": "agents-sdk-scaffold",
"description": "Scaffold an OpenAI Agents SDK app with a triage agent, handoffs, input/output/tool guardrails, session store, and a trace processor.",
"tags": [
"openai",
"agents-sdk",
"handoffs",
"guardrails",
"tracing",
"session"
],
"phase": 14,
"lesson": 16,
"lessonPath": "phases/14-agent-engineering/16-openai-agents-sdk",
"file": "phases/14-agent-engineering/16-openai-agents-sdk/outputs/skill-agents-sdk-scaffold.md"
},
{
"kind": "skill",
"name": "claude-agent-scaffold",
"description": "Scaffold a Claude Agent SDK app with subagents, lifecycle hooks, session store, MCP server attachment, and W3C trace propagation.",
"tags": [
"claude-agent-sdk",
"subagents",
"hooks",
"session-store",
"mcp"
],
"phase": 14,
"lesson": 17,
"lessonPath": "phases/14-agent-engineering/17-claude-agent-sdk",
"file": "phases/14-agent-engineering/17-claude-agent-sdk/outputs/skill-claude-agent-scaffold.md"
},
{
"kind": "skill",
"name": "runtime-picker",
"description": "Pick a production agent runtime (Agno, Mastra, LangGraph, provider SDK) for a given stack, latency budget, and operational shape.",
"tags": [
"agno",
"mastra",
"langgraph",
"runtime",
"selection"
],
"phase": 14,
"lesson": 18,
"lessonPath": "phases/14-agent-engineering/18-agno-and-mastra-runtimes",
"file": "phases/14-agent-engineering/18-agno-and-mastra-runtimes/outputs/skill-runtime-picker.md"
},
{
"kind": "skill",
"name": "benchmark-harness",
"description": "Build a SWE-bench-style harness for a codebase with FAIL_TO_PASS / PASS_TO_PASS gating, contamination checks, and step-count metrics.",
"tags": [
"swe-bench",
"gaia",
"agentbench",
"harness",
"evaluation"
],
"phase": 14,
"lesson": 19,
"lessonPath": "phases/14-agent-engineering/19-benchmarks-swebench-gaia",
"file": "phases/14-agent-engineering/19-benchmarks-swebench-gaia/outputs/skill-benchmark-harness.md"
},
{
"kind": "skill",
"name": "web-desktop-harness",
"description": "Build a WebArena/OSWorld-style harness with execution-based evaluation and trajectory-efficiency metrics.",
"tags": [
"webarena",
"osworld",
"harness",
"trajectory-efficiency"
],
"phase": 14,
"lesson": 20,
"lessonPath": "phases/14-agent-engineering/20-benchmarks-webarena-osworld",
"file": "phases/14-agent-engineering/20-benchmarks-webarena-osworld/outputs/skill-web-desktop-harness.md"
},
{
"kind": "skill",
"name": "computer-use-safety",
"description": "Build per-step safety classifier + confirmation gate for a computer-use agent, with allowlist navigation and injection-marker filtering.",
"tags": [
"computer-use",
"safety",
"claude",
"openai-cua",
"gemini"
],
"phase": 14,
"lesson": 21,
"lessonPath": "phases/14-agent-engineering/21-computer-use-agents",
"file": "phases/14-agent-engineering/21-computer-use-agents/outputs/skill-computer-use-safety.md"
},
{
"kind": "skill",
"name": "voice-pipeline",
"description": "Scaffold a Pipecat-shaped voice pipeline (VAD + STT + LLM + TTS + transport) with barge-in, confidence gating, and latency budget enforcement.",
"tags": [
"voice",
"pipecat",
"livekit",
"webrtc",
"latency"
],
"phase": 14,
"lesson": 22,
"lessonPath": "phases/14-agent-engineering/22-voice-agents-pipecat-livekit",
"file": "phases/14-agent-engineering/22-voice-agents-pipecat-livekit/outputs/skill-voice-pipeline.md"
},
{
"kind": "skill",
"name": "otel-genai",
"description": "Instrument an agent with OpenTelemetry GenAI semantic conventions — invoke_agent, chat, tool_call spans with correct attributes and opt-in content capture.",
"tags": [
"opentelemetry",
"genai",
"observability",
"tracing",
"semantic-conventions"
],
"phase": 14,
"lesson": 23,
"lessonPath": "phases/14-agent-engineering/23-otel-genai-conventions",
"file": "phases/14-agent-engineering/23-otel-genai-conventions/outputs/skill-otel-genai.md"
},
{
"kind": "skill",
"name": "obs-platform-wiring",
"description": "Pick an observability platform (Langfuse, Phoenix, Opik, Datadog) and wire traces + evals + prompt versions into an existing agent.",
"tags": [
"observability",
"langfuse",
"phoenix",
"opik",
"datadog",
"tracing"
],
"phase": 14,
"lesson": 24,
"lessonPath": "phases/14-agent-engineering/24-agent-observability-platforms",
"file": "phases/14-agent-engineering/24-agent-observability-platforms/outputs/skill-obs-platform-wiring.md"
},
{
"kind": "skill",
"name": "debate",
"description": "Scaffold a multi-agent debate with N debaters, R rounds, configurable topology (full mesh, star, ring), and a convergence rule.",
"tags": [
"debate",
"multi-agent",
"society-of-minds",
"sparse-topology"
],
"phase": 14,
"lesson": 25,
"lessonPath": "phases/14-agent-engineering/25-multi-agent-debate",
"file": "phases/14-agent-engineering/25-multi-agent-debate/outputs/skill-debate.md"
},
{
"kind": "skill",
"name": "failure-detector",
"description": "Generate failure-mode detectors for agent traces, wired to a trace store, tagging the five industry-recurring modes plus domain-specific signatures.",
"tags": [
"failure-modes",
"masft",
"detection",
"observability"
],
"phase": 14,
"lesson": 26,
"lessonPath": "phases/14-agent-engineering/26-failure-modes-agentic",
"file": "phases/14-agent-engineering/26-failure-modes-agentic/outputs/skill-failure-detector.md"
},
{
"kind": "skill",
"name": "injection-defense",
"description": "Build a PVE (Prompt-Validator-Executor) layer with source-tagged content, injection-marker scanning, and allowlist navigation for any agent runtime.",
"tags": [
"security",
"prompt-injection",
"pve",
"greshake",
"source-tag"
],
"phase": 14,
"lesson": 27,
"lessonPath": "phases/14-agent-engineering/27-prompt-injection-defense",
"file": "phases/14-agent-engineering/27-prompt-injection-defense/outputs/skill-injection-defense.md"
},
{
"kind": "skill",
"name": "orchestration-picker",
"description": "Pick an orchestration topology (supervisor, swarm, hierarchical, debate, or none) for a given problem and implement it minimally.",
"tags": [
"orchestration",
"supervisor",
"swarm",
"hierarchical",
"debate"
],
"phase": 14,
"lesson": 28,
"lessonPath": "phases/14-agent-engineering/28-orchestration-patterns",
"file": "phases/14-agent-engineering/28-orchestration-patterns/outputs/skill-orchestration-picker.md"
},
{
"kind": "skill",
"name": "runtime-shape",
"description": "Pick a production runtime shape (request-response, streaming, queue, event, cron, durable) and wire observability.",
"tags": [
"production",
"runtime",
"queue",
"event",
"durable",
"observability"
],
"phase": 14,
"lesson": 29,
"lessonPath": "phases/14-agent-engineering/29-production-runtimes",
"file": "phases/14-agent-engineering/29-production-runtimes/outputs/skill-runtime-shape.md"
},
{
"kind": "skill",
"name": "eval-suite",
"description": "Build a three-layer eval suite (static benchmarks, custom offline, online production) with evaluator-optimizer loop and CI gates.",
"tags": [
"evaluation",
"ci",
"regression",
"benchmarks",
"llm-judge"
],
"phase": 14,
"lesson": 30,
"lessonPath": "phases/14-agent-engineering/30-eval-driven-agent-development",
"file": "phases/14-agent-engineering/30-eval-driven-agent-development/outputs/skill-eval-suite.md"
},
{
"kind": "skill",
"name": "workbench-audit",
"description": "Audit a repo for the seven agent workbench surfaces and report which are missing, partial, or healthy before any agent work begins.",
"tags": [
"workbench",
"audit",
"reliability",
"agent-engineering"
],
"phase": 14,
"lesson": 31,
"lessonPath": "phases/14-agent-engineering/31-agent-workbench-why-models-fail",
"file": "phases/14-agent-engineering/31-agent-workbench-why-models-fail/outputs/skill-workbench-audit.md"
},
{
"kind": "mission",
"name": "Mission - Agent Workbench: Why Capable Models Still Fail",
"description": "",
"tags": [],
"phase": 14,
"lesson": 31,
"lessonPath": "phases/14-agent-engineering/31-agent-workbench-why-models-fail",
"file": "phases/14-agent-engineering/31-agent-workbench-why-models-fail/mission.md"
},
{
"kind": "skill",
"name": "minimal-workbench",
"description": "Lay down the three-file minimum viable agent workbench for any repo — short AGENTS.md router, durable agent_state.json, and a JSON task_board.json keyed to the project's current backlog.",
"tags": [
"workbench",
"agents-md",
"state",
"task-board",
"scaffold"
],
"phase": 14,
"lesson": 32,
"lessonPath": "phases/14-agent-engineering/32-minimal-agent-workbench",
"file": "phases/14-agent-engineering/32-minimal-agent-workbench/outputs/skill-minimal-workbench.md"
},
{
"kind": "mission",
"name": "Mission - The Minimal Agent Workbench",
"description": "",
"tags": [],
"phase": 14,
"lesson": 32,
"lessonPath": "phases/14-agent-engineering/32-minimal-agent-workbench",
"file": "phases/14-agent-engineering/32-minimal-agent-workbench/mission.md"
},
{
"kind": "skill",
"name": "rule-set-builder",
"description": "Interview a project owner, classify their existing prose instructions into five operational categories, and emit a versioned agent-rules.md plus a Python checker stub.",
"tags": [
"rules",
"instructions",
"constraints",
"checker",
"workbench"
],
"phase": 14,
"lesson": 33,
"lessonPath": "phases/14-agent-engineering/33-instructions-as-executable-constraints",
"file": "phases/14-agent-engineering/33-instructions-as-executable-constraints/outputs/skill-rule-set-builder.md"
},
{
"kind": "mission",
"name": "Mission - Agent Instructions as Executable Constraints",
"description": "",
"tags": [],
"phase": 14,
"lesson": 33,
"lessonPath": "phases/14-agent-engineering/33-instructions-as-executable-constraints",
"file": "phases/14-agent-engineering/33-instructions-as-executable-constraints/mission.md"
},
{
"kind": "skill",
"name": "state-schema",
"description": "Generate project-specific JSON Schemas for agent state and task board, a Python StateManager with atomic writes, and a migration scaffold so schema bumps cannot corrupt the workbench.",
"tags": [
"state",
"schema",
"json-schema",
"atomic-writes",
"migrations"
],
"phase": 14,
"lesson": 34,
"lessonPath": "phases/14-agent-engineering/34-repo-memory-and-state",
"file": "phases/14-agent-engineering/34-repo-memory-and-state/outputs/skill-state-schema.md"
},
{
"kind": "mission",
"name": "Mission - Repo Memory and Durable State",
"description": "",
"tags": [],
"phase": 14,
"lesson": 34,
"lessonPath": "phases/14-agent-engineering/34-repo-memory-and-state",
"file": "phases/14-agent-engineering/34-repo-memory-and-state/mission.md"
},
{
"kind": "skill",
"name": "init-script",
"description": "Interview a project and emit a deterministic init_agent.py with five probes plus a CI workflow that refuses to launch the agent if any probe fails.",
"tags": [
"init",
"probes",
"ci",
"workbench",
"fail-loud"
],
"phase": 14,
"lesson": 35,
"lessonPath": "phases/14-agent-engineering/35-initialization-scripts",
"file": "phases/14-agent-engineering/35-initialization-scripts/outputs/skill-init-script.md"
},
{
"kind": "mission",
"name": "Mission - Initialization Scripts for Agents",
"description": "",
"tags": [],
"phase": 14,
"lesson": 35,
"lessonPath": "phases/14-agent-engineering/35-initialization-scripts",
"file": "phases/14-agent-engineering/35-initialization-scripts/mission.md"
},
{
"kind": "skill",
"name": "scope-contract",
"description": "Generate per-task scope contracts with allowed/forbidden globs, acceptance criteria, and rollback plan, plus a CI-ready glob-aware checker that runs on every agent diff.",
"tags": [
"scope",
"contract",
"globs",
"diff-check",
"ci"
],
"phase": 14,
"lesson": 36,
"lessonPath": "phases/14-agent-engineering/36-scope-contracts",
"file": "phases/14-agent-engineering/36-scope-contracts/outputs/skill-scope-contract.md"
},
{
"kind": "mission",
"name": "Mission - Scope Contracts and Task Boundaries",
"description": "",
"tags": [],
"phase": 14,
"lesson": 36,
"lessonPath": "phases/14-agent-engineering/36-scope-contracts",
"file": "phases/14-agent-engineering/36-scope-contracts/mission.md"
},
{
"kind": "skill",
"name": "feedback-runner",
"description": "Wrap shell commands with deterministic stdout/stderr/exit/duration capture, persist a JSONL record per command, and refuse to advance the agent loop when feedback is missing.",
"tags": [
"feedback",
"subprocess",
"runner",
"jsonl",
"loop-control"
],
"phase": 14,
"lesson": 37,
"lessonPath": "phases/14-agent-engineering/37-runtime-feedback-loops",
"file": "phases/14-agent-engineering/37-runtime-feedback-loops/outputs/skill-feedback-runner.md"
},
{
"kind": "mission",
"name": "Mission - Runtime Feedback Loops",
"description": "",
"tags": [],
"phase": 14,
"lesson": 37,
"lessonPath": "phases/14-agent-engineering/37-runtime-feedback-loops",
"file": "phases/14-agent-engineering/37-runtime-feedback-loops/mission.md"
},
{
"kind": "skill",
"name": "verification-gate",
"description": "Generate a deterministic verification gate that combines scope, rule, and feedback artifacts into a single verification_report.json per task, plus CI wiring that refuses to merge without a green verdict.",
"tags": [
"verification",
"gate",
"deterministic",
"ci",
"override-log"
],
"phase": 14,
"lesson": 38,
"lessonPath": "phases/14-agent-engineering/38-verification-gates",
"file": "phases/14-agent-engineering/38-verification-gates/outputs/skill-verification-gate.md"
},
{
"kind": "mission",
"name": "Mission - Verification Gates",
"description": "",
"tags": [],
"phase": 14,
"lesson": 38,
"lessonPath": "phases/14-agent-engineering/38-verification-gates",
"file": "phases/14-agent-engineering/38-verification-gates/mission.md"
},
{
"kind": "skill",
"name": "reviewer-agent",
"description": "Stand up a reviewer agent role with a five-dimension rubric that reads builder artifacts, produces a structured review report, and starts human review from a written page instead of a blank one.",
"tags": [
"reviewer",
"rubric",
"role-separation",
"second-loop",
"review-report"
],
"phase": 14,
"lesson": 39,
"lessonPath": "phases/14-agent-engineering/39-reviewer-agent",
"file": "phases/14-agent-engineering/39-reviewer-agent/outputs/skill-reviewer-agent.md"
},
{
"kind": "mission",
"name": "Mission - Reviewer Agent: Separate Builder from Marker",
"description": "",
"tags": [],
"phase": 14,
"lesson": 39,
"lessonPath": "phases/14-agent-engineering/39-reviewer-agent",
"file": "phases/14-agent-engineering/39-reviewer-agent/mission.md"
},
{
"kind": "skill",
"name": "handoff-generator",
"description": "Generate end-of-session handoff packets from workbench artifacts, producing both human-readable Markdown and machine-readable JSON keyed to the seven canonical fields.",
"tags": [
"handoff",
"generator",
"session-end",
"packet",
"next-action"
],
"phase": 14,
"lesson": 40,
"lessonPath": "phases/14-agent-engineering/40-multi-session-handoff",
"file": "phases/14-agent-engineering/40-multi-session-handoff/outputs/skill-handoff-generator.md"
},
{
"kind": "mission",
"name": "Mission - Multi-Session Handoff",
"description": "",
"tags": [],
"phase": 14,
"lesson": 40,
"lessonPath": "phases/14-agent-engineering/40-multi-session-handoff",
"file": "phases/14-agent-engineering/40-multi-session-handoff/mission.md"
},
{
"kind": "skill",
"name": "workbench-benchmark",
"description": "Run the same task through prompt-only and workbench-guided pipelines on a project's own sample app and emit a five-outcome before/after report.",
"tags": [
"benchmark",
"before-after",
"evaluation",
"workbench",
"sample-app"
],
"phase": 14,
"lesson": 41,
"lessonPath": "phases/14-agent-engineering/41-workbench-for-real-repos",
"file": "phases/14-agent-engineering/41-workbench-for-real-repos/outputs/skill-workbench-benchmark.md"
},
{
"kind": "mission",
"name": "Mission - The Workbench on a Real Repo",
"description": "",
"tags": [],
"phase": 14,
"lesson": 41,
"lessonPath": "phases/14-agent-engineering/41-workbench-for-real-repos",
"file": "phases/14-agent-engineering/41-workbench-for-real-repos/mission.md"
},
{
"kind": "skill",
"name": "workbench-pack",
"description": "Generate a project-tuned drop-in agent workbench pack — rules sharpened to the team's history, scope globs matched to the repo, rubric dimensions extended with one domain-specific entry.",
"tags": [
"capstone",
"workbench-pack",
"installer",
"schemas",
"drop-in"
],
"phase": 14,
"lesson": 42,
"lessonPath": "phases/14-agent-engineering/42-agent-workbench-capstone",
"file": "phases/14-agent-engineering/42-agent-workbench-capstone/outputs/skill-workbench-pack.md"
},
{
"kind": "mission",
"name": "Mission - Capstone: Ship a Reusable Agent Workbench Pack",
"description": "",
"tags": [],
"phase": 14,
"lesson": 42,
"lessonPath": "phases/14-agent-engineering/42-agent-workbench-capstone",
"file": "phases/14-agent-engineering/42-agent-workbench-capstone/mission.md"
},
{
"kind": "skill",
"name": "horizon-reality-check",
"description": "Given a task you want to hand to an agent, decide whether the current frontier's horizon covers it with enough margin.",
"tags": [
"autonomous-agents",
"metr",
"time-horizon",
"reliability",
"deployment"
],
"phase": 15,
"lesson": 1,
"lessonPath": "phases/15-autonomous-systems/01-long-horizon-agents",
"file": "phases/15-autonomous-systems/01-long-horizon-agents/outputs/skill-horizon-reality-check.md"
},
{
"kind": "skill",
"name": "star-loop-reviewer",
"description": "Audit a proposed self-taught reasoning pipeline (STaR-family) before you commit training compute to it.",
"tags": [
"star",
"vstar",
"quiet-star",
"self-improvement",
"reasoning",
"bootstrap"
],
"phase": 15,
"lesson": 2,
"lessonPath": "phases/15-autonomous-systems/02-star-family-reasoning",
"file": "phases/15-autonomous-systems/02-star-family-reasoning/outputs/skill-star-loop-reviewer.md"
},
{
"kind": "skill",
"name": "evaluator-rigor-audit",
"description": "Audit a proposed AlphaEvolve-style evolutionary coding loop's evaluator before committing any compute to the search.",
"tags": [
"alphaevolve",
"evolutionary-coding",
"evaluator",
"reward-hacking",
"deepmind"
],
"phase": 15,
"lesson": 3,
"lessonPath": "phases/15-autonomous-systems/03-alphaevolve-evolutionary-coding",
"file": "phases/15-autonomous-systems/03-alphaevolve-evolutionary-coding/outputs/skill-evaluator-rigor-audit.md"
},
{
"kind": "skill",
"name": "dgm-evaluator-firewall",
"description": "Specify the evaluator separation a Darwin-Godel-Machine-style self-modifying agent loop needs to avoid documented reward hacking.",
"tags": [
"dgm",
"self-modification",
"reward-hacking",
"evaluator",
"sandbox"
],
"phase": 15,
"lesson": 4,
"lessonPath": "phases/15-autonomous-systems/04-darwin-godel-machine",
"file": "phases/15-autonomous-systems/04-darwin-godel-machine/outputs/skill-dgm-evaluator-firewall.md"
},
{
"kind": "skill",
"name": "ai-scientist-sandbox-review",
"description": "Two-gate review checklist for research-loop agent outputs before anything leaves the sandbox.",
"tags": [
"ai-scientist",
"research-agent",
"sandbox",
"peer-review",
"disclosure"
],
"phase": 15,
"lesson": 5,
"lessonPath": "phases/15-autonomous-systems/05-ai-scientist-v2",
"file": "phases/15-autonomous-systems/05-ai-scientist-v2/outputs/skill-ai-scientist-sandbox-review.md"
},
{
"kind": "skill",
"name": "aar-deployment-review",
"description": "Pre-deployment review of an automated-alignment-research pipeline, including sandbox isolation and log integrity.",
"tags": [
"aar",
"alignment-research",
"sandbox",
"log-integrity",
"rsp"
],
"phase": 15,
"lesson": 6,
"lessonPath": "phases/15-autonomous-systems/06-automated-alignment-research",
"file": "phases/15-autonomous-systems/06-automated-alignment-research/outputs/skill-aar-deployment-review.md"
},
{
"kind": "skill",
"name": "rsi-cycle-pause-spec",
"description": "Specify the conditions under which an RSI pipeline must pause and wait for human review before the next cycle.",
"tags": [
"rsi",
"self-improvement",
"alignment",
"pause-threshold",
"rsp"
],
"phase": 15,
"lesson": 7,
"lessonPath": "phases/15-autonomous-systems/07-recursive-self-improvement",
"file": "phases/15-autonomous-systems/07-recursive-self-improvement/outputs/skill-rsi-cycle-pause-spec.md"
},
{
"kind": "skill",
"name": "bounded-loop-review",
"description": "Audit a proposed bounded self-improvement loop against the four-primitive stack (invariants, anchor, multi-objective, regression detection).",
"tags": [
"bounded-self-improvement",
"invariants",
"alignment-anchor",
"rsi-safety"
],
"phase": 15,
"lesson": 8,
"lessonPath": "phases/15-autonomous-systems/08-bounded-self-improvement",
"file": "phases/15-autonomous-systems/08-bounded-self-improvement/outputs/skill-bounded-loop-review.md"
},
{
"kind": "skill",
"name": "coding-scaffold-audit",
"description": "Audit a proposed coding-agent scaffold (retrieval, verifier loop, sandbox, benchmark fit) before adopting it for production code changes.",
"tags": [
"coding-agent",
"scaffolding",
"swe-bench",
"codeact",
"openhands"
],
"phase": 15,
"lesson": 9,
"lessonPath": "phases/15-autonomous-systems/09-coding-agent-landscape",
"file": "phases/15-autonomous-systems/09-coding-agent-landscape/outputs/skill-scaffold-audit.md"
},
{
"kind": "skill",
"name": "permission-mode-picker",
"description": "Match a Claude Code task to the correct permission mode, budget caps, and required isolation before starting a run.",
"tags": [
"claude-code",
"permission-modes",
"auto-mode",
"budgets",
"isolation"
],
"phase": 15,
"lesson": 10,
"lessonPath": "phases/15-autonomous-systems/10-claude-code-permission-modes",
"file": "phases/15-autonomous-systems/10-claude-code-permission-modes/outputs/skill-permission-mode-picker.md"
},
{
"kind": "skill",
"name": "browser-agent-trust-boundary",
"description": "Scope a proposed browser-agent deployment — trust zones, authorized writes, required defenses — before the agent touches a real site.",
"tags": [
"browser-agents",
"prompt-injection",
"trust-boundary",
"osworld",
"webarena"
],
"phase": 15,
"lesson": 11,
"lessonPath": "phases/15-autonomous-systems/11-browser-agents",
"file": "phases/15-autonomous-systems/11-browser-agents/outputs/skill-browser-agent-trust-boundary.md"
},
{
"kind": "skill",
"name": "durable-execution-review",
"description": "Review a proposed long-running agent deployment for correct durable-execution shape (activities, determinism, checkpoint backend, human-input state, HITL-on-resume).",
"tags": [
"durable-execution",
"workflows",
"checkpointing",
"temporal",
"langgraph",
"agents-sdk"
],
"phase": 15,
"lesson": 12,
"lessonPath": "phases/15-autonomous-systems/12-durable-execution",
"file": "phases/15-autonomous-systems/12-durable-execution/outputs/skill-durable-execution-review.md"
},
{
"kind": "skill",
"name": "agent-budget-audit",
"description": "Audit an agent deployment's cost-governor stack and flag missing layers before enabling unattended runs.",
"tags": [
"cost-governors",
"denial-of-wallet",
"budgets",
"claude-code-sdk",
"agent-governance"
],
"phase": 15,
"lesson": 13,
"lessonPath": "phases/15-autonomous-systems/13-cost-governors",
"file": "phases/15-autonomous-systems/13-cost-governors/outputs/skill-agent-budget-audit.md"
},
{
"kind": "skill",
"name": "tripwire-design",
"description": "Review a proposed agent detector stack (kill switch, circuit breakers, canary tokens) and flag missing tripwires before the first autonomous run.",
"tags": [
"kill-switch",
"circuit-breaker",
"canary",
"honeytoken",
"detection-and-response"
],
"phase": 15,
"lesson": 14,
"lessonPath": "phases/15-autonomous-systems/14-kill-switches-canaries",
"file": "phases/15-autonomous-systems/14-kill-switches-canaries/outputs/skill-tripwire-design.md"
},
{
"kind": "skill",
"name": "hitl-design",
"description": "Review a proposed Human-in-the-Loop workflow for propose-then-commit shape and flag missing metadata, idempotency, verification, or challenge-and-response layers.",
"tags": [
"hitl",
"propose-then-commit",
"idempotency",
"langgraph",
"cloudflare",
"agent-framework",
"eu-ai-act"
],
"phase": 15,
"lesson": 15,
"lessonPath": "phases/15-autonomous-systems/15-propose-then-commit",
"file": "phases/15-autonomous-systems/15-propose-then-commit/outputs/skill-hitl-design.md"
},
{
"kind": "skill",
"name": "rollback-rehearsal",
"description": "Design a rollback-rehearsal test for a proposed autonomous workflow and audit the checkpoint backend for audit-trail persistence.",
"tags": [
"checkpointing",
"rollback",
"idempotency",
"eu-ai-act-article-14",
"durable-execution"
],
"phase": 15,
"lesson": 16,
"lessonPath": "phases/15-autonomous-systems/16-checkpoints-rollback",
"file": "phases/15-autonomous-systems/16-checkpoints-rollback/outputs/skill-rollback-rehearsal.md"
},
{
"kind": "skill",
"name": "constitution-review",
"description": "Audit a deployment's constitutional layer — hardcoded prohibitions, soft-coded defaults, operator-adjustable bounds, and four-tier hierarchy resolution.",
"tags": [
"constitutional-ai",
"rule-override",
"hierarchy",
"cai",
"rlaif",
"hardcoded-prohibition"
],
"phase": 15,
"lesson": 17,
"lessonPath": "phases/15-autonomous-systems/17-constitutional-ai",
"file": "phases/15-autonomous-systems/17-constitutional-ai/outputs/skill-constitution-review.md"
},
{
"kind": "skill",
"name": "classifier-stack-audit",
"description": "Audit a deployment's input/output classifier stack (model, taxonomy, input rails, output rails, dialog rails) and flag adversarial-attack gaps.",
"tags": [
"llama-guard",
"nemo-guardrails",
"input-rails",
"output-rails",
"colang",
"adversarial-attacks"
],
"phase": 15,
"lesson": 18,
"lessonPath": "phases/15-autonomous-systems/18-llama-guard",
"file": "phases/15-autonomous-systems/18-llama-guard/outputs/skill-classifier-stack-audit.md"
},
{
"kind": "skill",
"name": "scaling-policy-review",
"description": "Review a frontier-lab scaling policy (Anthropic RSP, OpenAI Preparedness, DeepMind FSF, internal) against the RSP v3.0 reference shape.",
"tags": [
"rsp",
"scaling-policy",
"ai-rd-4",
"pause-commitment",
"saferai",
"governance"
],
"phase": 15,
"lesson": 19,
"lessonPath": "phases/15-autonomous-systems/19-anthropic-rsp",
"file": "phases/15-autonomous-systems/19-anthropic-rsp/outputs/skill-scaling-policy-review.md"
},
{
"kind": "skill",
"name": "cross-policy-diff",
"description": "Produce a cross-policy comparison for a specific capability using the OpenAI Preparedness Framework v2, Anthropic RSP v3.0, and DeepMind FSF v3 as reference.",
"tags": [
"preparedness-framework",
"fsf",
"rsp",
"cross-policy",
"scaling-policy"
],
"phase": 15,
"lesson": 20,
"lessonPath": "phases/15-autonomous-systems/20-openai-preparedness-deepmind-fsf",
"file": "phases/15-autonomous-systems/20-openai-preparedness-deepmind-fsf/outputs/skill-cross-policy-diff.md"
},
{
"kind": "skill",
"name": "horizon-interpretation",
"description": "Review a vendor's time-horizon claim and produce a gap analysis between benchmark claim and deployment reality.",
"tags": [
"metr",
"time-horizon",
"hcast",
"re-bench",
"eval-vs-deploy",
"external-evaluation"
],
"phase": 15,
"lesson": 21,
"lessonPath": "phases/15-autonomous-systems/21-metr-external-evaluation",
"file": "phases/15-autonomous-systems/21-metr-external-evaluation/outputs/skill-horizon-interpretation.md"
},
{
"kind": "skill",
"name": "societal-risk-review",
"description": "Review a deployment for societal-scale-risk posture using the CAIS four-risk framework and CAISI / SB-53 regulatory context.",
"tags": [
"cais",
"caisi",
"four-risk-framework",
"organizational-risk",
"sb-53",
"societal-risk"
],
"phase": 15,
"lesson": 22,
"lessonPath": "phases/15-autonomous-systems/22-cais-caisi-societal-risk",
"file": "phases/15-autonomous-systems/22-cais-caisi-societal-risk/outputs/skill-societal-risk-review.md"
},
{
"kind": "prompt",
"name": "prompt-multi-agent-decision",
"description": "Decide whether a task needs a multi-agent system or a single agent",
"tags": [],
"phase": 16,
"lesson": 1,
"lessonPath": "phases/16-multi-agent-and-swarms/01-why-multi-agent",
"file": "phases/16-multi-agent-and-swarms/01-why-multi-agent/outputs/prompt-multi-agent-decision.md"
},
{
"kind": "skill",
"name": "fipa-mapper",
"description": "Map any 2026 agent-protocol spec (MCP, A2A, ACP, ANP, CA-MCP, NLIP, or a new one) onto FIPA-ACL performatives and interaction protocols to decide what is genuine novelty and what is reinvention.",
"tags": [
"multi-agent",
"protocols",
"FIPA",
"speech-acts",
"interoperability"
],
"phase": 16,
"lesson": 2,
"lessonPath": "phases/16-multi-agent-and-swarms/02-fipa-acl-heritage",
"file": "phases/16-multi-agent-and-swarms/02-fipa-acl-heritage/outputs/skill-fipa-mapper.md"
},
{
"kind": "prompt",
"name": "prompt-protocol-selector",
"description": "Helps choose the right agent communication protocol (MCP, A2A, ACP, ANP) based on system requirements",
"tags": [],
"phase": 16,
"lesson": 3,
"lessonPath": "phases/16-multi-agent-and-swarms/03-communication-protocols",
"file": "phases/16-multi-agent-and-swarms/03-communication-protocols/outputs/prompt-protocol-selector.md"
},
{
"kind": "skill",
"name": "primitive-mapper",
"description": "Map any multi-agent framework or codebase to the four primitive axes (agent, handoff, shared state, orchestrator).",
"tags": [
"multi-agent",
"primitives",
"framework-comparison",
"architecture"
],
"phase": 16,
"lesson": 4,
"lessonPath": "phases/16-multi-agent-and-swarms/04-primitive-model",
"file": "phases/16-multi-agent-and-swarms/04-primitive-model/outputs/skill-primitive-mapper.md"
},
{
"kind": "skill",
"name": "supervisor-designer",
"description": "Design a supervisor/orchestrator-worker system for a given research-style query, specifying lead prompt, worker roles, decomposition rules, and synthesis template.",
"tags": [
"multi-agent",
"supervisor",
"orchestrator",
"anthropic-research",
"langgraph"
],
"phase": 16,
"lesson": 5,
"lessonPath": "phases/16-multi-agent-and-swarms/05-supervisor-orchestrator-pattern",
"file": "phases/16-multi-agent-and-swarms/05-supervisor-orchestrator-pattern/outputs/skill-supervisor-designer.md"
},
{
"kind": "skill",
"name": "hierarchy-fitness",
"description": "Decide whether a multi-agent task fits hierarchical, flat supervisor, or sequential. Surface the failure modes that matter.",
"tags": [
"multi-agent",
"hierarchy",
"crewai",
"langgraph",
"decomposition-drift"
],
"phase": 16,
"lesson": 6,
"lessonPath": "phases/16-multi-agent-and-swarms/06-hierarchical-architecture",
"file": "phases/16-multi-agent-and-swarms/06-hierarchical-architecture/outputs/skill-hierarchy-fitness.md"
},
{
"kind": "skill",
"name": "debate-configurator",
"description": "Configure a multi-agent debate for a given task, estimating quality gain and token cost before running.",
"tags": [
"multi-agent",
"debate",
"society-of-mind",
"consensus"
],
"phase": 16,
"lesson": 7,
"lessonPath": "phases/16-multi-agent-and-swarms/07-society-of-mind-debate",
"file": "phases/16-multi-agent-and-swarms/07-society-of-mind-debate/outputs/skill-debate-configurator.md"
},
{
"kind": "skill",
"name": "role-designer",
"description": "Produce a role roster for a multi-agent system, naming the planner/executor/critic/verifier for a given task with explicit I/O schemas.",
"tags": [
"multi-agent",
"role-specialization",
"metagpt",
"chatdev",
"verification"
],
"phase": 16,
"lesson": 8,
"lessonPath": "phases/16-multi-agent-and-swarms/08-role-specialization",
"file": "phases/16-multi-agent-and-swarms/08-role-specialization/outputs/skill-role-designer.md"
},
{
"kind": "skill",
"name": "swarm-fit",
"description": "Decide whether a task fits a swarm (decentralized) architecture or a supervisor (centralized) one.",
"tags": [
"multi-agent",
"swarm",
"decentralized",
"langgraph",
"matrix"
],
"phase": 16,
"lesson": 9,
"lessonPath": "phases/16-multi-agent-and-swarms/09-parallel-swarm-networks",
"file": "phases/16-multi-agent-and-swarms/09-parallel-swarm-networks/outputs/skill-swarm-fit.md"
},
{
"kind": "skill",
"name": "groupchat-selector",
"description": "Configure an AutoGen/AG2-style GroupChat selector for a task, naming the selector variant, termination, and anti-hot-speaker rules.",
"tags": [
"multi-agent",
"groupchat",
"autogen",
"ag2",
"speaker-selection"
],
"phase": 16,
"lesson": 10,
"lessonPath": "phases/16-multi-agent-and-swarms/10-group-chat-speaker-selection",
"file": "phases/16-multi-agent-and-swarms/10-group-chat-speaker-selection/outputs/skill-groupchat-selector.md"
},
{
"kind": "skill",
"name": "handoff-designer",
"description": "Design a handoff topology for a Swarm/Agents-SDK-style system: which agents exist, which handoffs they can call, what context transfers.",
"tags": [
"multi-agent",
"swarm",
"handoff",
"openai-agents-sdk"
],
"phase": 16,
"lesson": 11,
"lessonPath": "phases/16-multi-agent-and-swarms/11-handoffs-and-routines",
"file": "phases/16-multi-agent-and-swarms/11-handoffs-and-routines/outputs/skill-handoff-designer.md"
},
{
"kind": "skill",
"name": "a2a-integrator",
"description": "Design an A2A integration between two agents — Agent Card, task schemas, auth, streaming or polling.",
"tags": [
"multi-agent",
"a2a",
"protocol",
"interoperability",
"google"
],
"phase": 16,
"lesson": 12,
"lessonPath": "phases/16-multi-agent-and-swarms/12-a2a-protocol",
"file": "phases/16-multi-agent-and-swarms/12-a2a-protocol/outputs/skill-a2a-integrator.md"
},
{
"kind": "skill",
"name": "memory-auditor",
"description": "Audit a multi-agent system's shared-memory design for provenance, versioning, verifier separation, and projection schema. Flag memory-poisoning exposure before production.",
"tags": [
"multi-agent",
"shared-state",
"blackboard",
"memory-poisoning",
"provenance"
],
"phase": 16,
"lesson": 13,
"lessonPath": "phases/16-multi-agent-and-swarms/13-shared-memory-blackboard",
"file": "phases/16-multi-agent-and-swarms/13-shared-memory-blackboard/outputs/skill-memory-auditor.md"
},
{
"kind": "skill",
"name": "consensus-designer",
"description": "Design a BFT-aware consensus protocol for a multi-agent ensemble. Picks clustering, weighting, threshold, and escalation policy; attack-tests the design against byzantine, sycophancy, and monoculture patterns.",
"tags": [
"multi-agent",
"consensus",
"BFT",
"voting",
"confidence"
],
"phase": 16,
"lesson": 14,
"lessonPath": "phases/16-multi-agent-and-swarms/14-consensus-and-bft",
"file": "phases/16-multi-agent-and-swarms/14-consensus-and-bft/outputs/skill-consensus-designer.md"
},
{
"kind": "skill",
"name": "topology-picker",
"description": "Pick a multi-agent debate topology (star / chain / tree / graph), an N of agents, a heterogeneity profile, and a round bound for a given task.",
"tags": [
"multi-agent",
"debate",
"topology",
"voting",
"self-consistency"
],
"phase": 16,
"lesson": 15,
"lessonPath": "phases/16-multi-agent-and-swarms/15-voting-debate-topology",
"file": "phases/16-multi-agent-and-swarms/15-voting-debate-topology/outputs/skill-topology-picker.md"
},
{
"kind": "skill",
"name": "bargainer-designer",
"description": "Design a negotiation protocol: which agent narrates, which component generates offers, how private scratchpads separate from public messages, what the round bound is, and how deal rate is monitored.",
"tags": [
"multi-agent",
"negotiation",
"bargaining",
"contract-net",
"OG-Narrator"
],
"phase": 16,
"lesson": 16,
"lessonPath": "phases/16-multi-agent-and-swarms/16-negotiation-bargaining",
"file": "phases/16-multi-agent-and-swarms/16-negotiation-bargaining/outputs/skill-bargainer-designer.md"
},
{
"kind": "skill",
"name": "simulation-designer",
"description": "Design a generative-agent simulation (Smallville-style) for a given scenario. Specifies memory schema, reflection cadence, plan horizon, spatial/social constraints, and evaluation metrics.",
"tags": [
"multi-agent",
"simulation",
"generative-agents",
"emergence",
"memory"
],
"phase": 16,
"lesson": 17,
"lessonPath": "phases/16-multi-agent-and-swarms/17-generative-agents-simulation",
"file": "phases/16-multi-agent-and-swarms/17-generative-agents-simulation/outputs/skill-simulation-designer.md"
},
{
"kind": "skill",
"name": "tom-auditor",
"description": "Audit a multi-agent system that claims \"emergent coordination.\" Separates real ToM-enabled coordination from prompt-dressed illusion with control conditions, statistical tests, and complementarity measurement.",
"tags": [
"multi-agent",
"theory-of-mind",
"coordination",
"evaluation",
"emergence"
],
"phase": 16,
"lesson": 18,
"lessonPath": "phases/16-multi-agent-and-swarms/18-theory-of-mind-coordination",
"file": "phases/16-multi-agent-and-swarms/18-theory-of-mind-coordination/outputs/skill-tom-auditor.md"
},
{
"kind": "skill",
"name": "swarm-optimizer",
"description": "Choose between PSO, ACO, genetic algorithms, and gradient-based optimizers for a given LLM or agent optimization problem. Bio-inspired swarm algorithms are gradient-free and suit LLM-era workloads where the search space is discrete or the fitness function is black-box.",
"tags": [
"multi-agent",
"swarm-optimization",
"PSO",
"ACO",
"prompt-optimization",
"routing"
],
"phase": 16,
"lesson": 19,
"lessonPath": "phases/16-multi-agent-and-swarms/19-swarm-optimization-pso-aco",
"file": "phases/16-multi-agent-and-swarms/19-swarm-optimization-pso-aco/outputs/skill-swarm-optimizer.md"
},
{
"kind": "skill",
"name": "marl-picker",
"description": "Choose a MARL algorithm (MADDPG, QMIX, MAPPO, IQL, or extensions) for a given multi-agent task. Consider cooperative vs competitive, action-space type, heterogeneity, reward structure, and scale.",
"tags": [
"multi-agent",
"MARL",
"MADDPG",
"QMIX",
"MAPPO",
"CTDE"
],
"phase": 16,
"lesson": 20,
"lessonPath": "phases/16-multi-agent-and-swarms/20-marl-maddpg-qmix-mappo",
"file": "phases/16-multi-agent-and-swarms/20-marl-maddpg-qmix-mappo/outputs/skill-marl-picker.md"
},
{
"kind": "skill",
"name": "economy-designer",
"description": "Design a minimal agent economy — identity, credit attribution, payment mechanism, reputation. Picks the smallest stack that solves the user's multi-agent incentive problem.",
"tags": [
"multi-agent",
"economy",
"Shapley",
"auctions",
"reputation",
"DePIN"
],
"phase": 16,
"lesson": 21,
"lessonPath": "phases/16-multi-agent-and-swarms/21-agent-economies",
"file": "phases/16-multi-agent-and-swarms/21-agent-economies/outputs/skill-economy-designer.md"
},
{
"kind": "skill",
"name": "scaling-advisor",
"description": "Advise on durable-execution choice for a multi-agent production system. Picks between FastAPI + Postgres, LangGraph runtime, Temporal, Restate, or custom based on concrete load and state-retention needs.",
"tags": [
"multi-agent",
"production",
"scaling",
"durable-execution",
"queues",
"checkpoints"
],
"phase": 16,
"lesson": 22,
"lessonPath": "phases/16-multi-agent-and-swarms/22-production-scaling-queues-checkpoints",
"file": "phases/16-multi-agent-and-swarms/22-production-scaling-queues-checkpoints/outputs/skill-scaling-advisor.md"
},
{
"kind": "skill",
"name": "mast-auditor",
"description": "Run a MAST-style failure-mode audit on a multi-agent system. Categorize execution-trace failures into Specification / Coordination / Verification and the Groupthink families; rank mitigations by expected failure reduction.",
"tags": [
"multi-agent",
"failure-modes",
"MAST",
"groupthink",
"circuit-breaker",
"audit"
],
"phase": 16,
"lesson": 23,
"lessonPath": "phases/16-multi-agent-and-swarms/23-failure-modes-mast-groupthink",
"file": "phases/16-multi-agent-and-swarms/23-failure-modes-mast-groupthink/outputs/skill-mast-auditor.md"
},
{
"kind": "skill",
"name": "benchmark-reader",
"description": "Read a multi-agent benchmark claim skeptically. Grades the claim on benchmark selection, contamination, baselines, statistical significance, task diversity, and cost disclosure.",
"tags": [
"multi-agent",
"benchmarks",
"evaluation",
"SWE-bench",
"MARBLE"
],
"phase": 16,
"lesson": 24,
"lessonPath": "phases/16-multi-agent-and-swarms/24-evaluation-coordination-benchmarks",
"file": "phases/16-multi-agent-and-swarms/24-evaluation-coordination-benchmarks/outputs/skill-benchmark-reader.md"
},
{
"kind": "skill",
"name": "case-study-mapper",
"description": "Map a proposed multi-agent system design to the closest 2026 production reference (Anthropic Research, MetaGPT/ChatDev, or OpenClaw/Moltbook). Surface known trade-offs, recommended framework, and the specific design decisions already tested in production.",
"tags": [
"multi-agent",
"case-studies",
"production",
"framework-selection",
"reference-architectures"
],
"phase": 16,
"lesson": 25,
"lessonPath": "phases/16-multi-agent-and-swarms/25-case-studies-2026-sota",
"file": "phases/16-multi-agent-and-swarms/25-case-studies-2026-sota/outputs/skill-case-study-mapper.md"
},
{
"kind": "skill",
"name": "managed-platform-picker",
"description": "Pick a managed LLM platform (Bedrock, Azure OpenAI, Vertex AI) and a second for redundancy, given workload, SLA, and compliance requirements — then produce a FinOps instrumentation plan.",
"tags": [
"bedrock",
"azure-openai",
"vertex-ai",
"ptu",
"finops",
"managed-platforms"
],
"phase": 17,
"lesson": 1,
"lessonPath": "phases/17-infrastructure-and-production/01-managed-llm-platforms",
"file": "phases/17-infrastructure-and-production/01-managed-llm-platforms/outputs/skill-managed-platform-picker.md"
},
{
"kind": "skill",
"name": "inference-platform-picker",
"description": "Pick an inference platform (Fireworks, Together, Baseten, Modal, Replicate, Anyscale, or custom silicon) given workload, SLA, budget, and operational constraints. Normalize per-token, per-minute, and per-prediction pricing.",
"tags": [
"inference",
"fireworks",
"together",
"baseten",
"modal",
"replicate",
"anyscale",
"economics"
],
"phase": 17,
"lesson": 2,
"lessonPath": "phases/17-infrastructure-and-production/02-inference-platform-economics",
"file": "phases/17-infrastructure-and-production/02-inference-platform-economics/outputs/skill-inference-platform-picker.md"
},
{
"kind": "skill",
"name": "gpu-autoscaler-plan",
"description": "Design a three-layer GPU autoscaling plan (Karpenter + KAI Scheduler + application signals) for a Kubernetes-based LLM serving cluster. Diagnose DCGM_FI_DEV_GPU_UTIL traps and partial-allocation failures.",
"tags": [
"kubernetes",
"gpu",
"autoscaling",
"karpenter",
"kai-scheduler",
"hpa",
"dynamo-planner",
"llm-d"
],
"phase": 17,
"lesson": 3,
"lessonPath": "phases/17-infrastructure-and-production/03-gpu-autoscaling-kubernetes",
"file": "phases/17-infrastructure-and-production/03-gpu-autoscaling-kubernetes/outputs/skill-gpu-autoscaler-plan.md"
},
{
"kind": "skill",
"name": "vllm-scheduler-reader",
"description": "Diagnose a vLLM serving config by reading the scheduler-level knobs and identifying which of PagedAttention, continuous batching, and chunked prefill is the bottleneck.",
"tags": [
"vllm",
"paged-attention",
"continuous-batching",
"chunked-prefill",
"serving",
"scheduler"
],
"phase": 17,
"lesson": 4,
"lessonPath": "phases/17-infrastructure-and-production/04-vllm-serving-internals",
"file": "phases/17-infrastructure-and-production/04-vllm-serving-internals/outputs/skill-vllm-scheduler-reader.md"
},
{
"kind": "skill",
"name": "eagle3-rollout",
"description": "Produce a staged EAGLE-3 speculative-decoding rollout plan that measures acceptance rate alpha on real traffic before shipping.",
"tags": [
"speculative-decoding",
"eagle-3",
"vllm",
"alpha",
"production-rollout"
],
"phase": 17,
"lesson": 5,
"lessonPath": "phases/17-infrastructure-and-production/05-eagle3-speculative-decoding",
"file": "phases/17-infrastructure-and-production/05-eagle3-speculative-decoding/outputs/skill-eagle3-rollout.md"
},
{
"kind": "skill",
"name": "radix-scheduler-advisor",
"description": "Advise on SGLang adoption and prompt-ordering discipline for prefix-heavy workloads that want RadixAttention's cache reuse.",
"tags": [
"sglang",
"radixattention",
"prefix-caching",
"scheduler",
"prompt-ordering"
],
"phase": 17,
"lesson": 6,
"lessonPath": "phases/17-infrastructure-and-production/06-sglang-radixattention",
"file": "phases/17-infrastructure-and-production/06-sglang-radixattention/outputs/skill-radix-scheduler-advisor.md"
},
{
"kind": "skill",
"name": "trtllm-blackwell-advisor",
"description": "Decide whether Blackwell + TensorRT-LLM + Dynamo is worth the NVIDIA-lock for a given workload and budget.",
"tags": [
"tensorrt-llm",
"blackwell",
"b200",
"gb200",
"nvfp4",
"fp8",
"dynamo"
],
"phase": 17,
"lesson": 7,
"lessonPath": "phases/17-infrastructure-and-production/07-tensorrt-llm-blackwell",
"file": "phases/17-infrastructure-and-production/07-tensorrt-llm-blackwell/outputs/skill-trtllm-blackwell-advisor.md"
},
{
"kind": "skill",
"name": "slo-goodput-gate",
"description": "Produce a CI/CD-ready benchmark recipe that gates LLM deploys on goodput, not throughput, with P50/P90/P99 percentiles and a documented tool choice.",
"tags": [
"inference-metrics",
"goodput",
"ttft",
"tpot",
"itl",
"slo",
"benchmarking"
],
"phase": 17,
"lesson": 8,
"lessonPath": "phases/17-infrastructure-and-production/08-inference-metrics-goodput",
"file": "phases/17-infrastructure-and-production/08-inference-metrics-goodput/outputs/skill-slo-goodput-gate.md"
},
{
"kind": "skill",
"name": "quantization-picker",
"description": "Pick a 2026 quantization format given hardware, engine, workload, and quality tolerance, and produce a calibration + validation plan.",
"tags": [
"quantization",
"awq",
"gptq",
"gguf",
"fp8",
"nvfp4",
"calibration"
],
"phase": 17,
"lesson": 9,
"lessonPath": "phases/17-infrastructure-and-production/09-production-quantization",
"file": "phases/17-infrastructure-and-production/09-production-quantization/outputs/skill-quantization-picker.md"
},
{
"kind": "skill",
"name": "cold-start-planner",
"description": "Pick and stack cold-start mitigations for serverless LLM deployments. Budget phases (node, image, weights, engine, first forward) and match mitigations to SLA.",
"tags": [
"cold-start",
"serverless",
"bottlerocket",
"model-streamer",
"gpu-snapshot",
"warm-pool",
"serverlessllm"
],
"phase": 17,
"lesson": 10,
"lessonPath": "phases/17-infrastructure-and-production/10-cold-start-mitigation",
"file": "phases/17-infrastructure-and-production/10-cold-start-mitigation/outputs/skill-cold-start-planner.md"
},
{
"kind": "skill",
"name": "multi-region-router",
"description": "Design a multi-region LLM routing plan with KV-cache locality, residency boundaries, DR manifest, and a quarterly failover drill.",
"tags": [
"multi-region",
"kv-cache",
"routing",
"dr",
"bedrock-cri",
"vllm-router",
"llm-d",
"gorgo"
],
"phase": 17,
"lesson": 11,
"lessonPath": "phases/17-infrastructure-and-production/11-multi-region-kv-locality",
"file": "phases/17-infrastructure-and-production/11-multi-region-kv-locality/outputs/skill-multi-region-router.md"
},
{
"kind": "skill",
"name": "edge-target-picker",
"description": "Pick an edge inference target (Apple ANE, Qualcomm Hexagon, WebGPU/WebLLM, NVIDIA Jetson) and matching quantization format given device, model, and latency budget.",
"tags": [
"edge",
"ane",
"hexagon",
"webgpu",
"webllm",
"jetson",
"core-ml",
"qnn",
"nvfp4"
],
"phase": 17,
"lesson": 12,
"lessonPath": "phases/17-infrastructure-and-production/12-edge-inference",
"file": "phases/17-infrastructure-and-production/12-edge-inference/outputs/skill-edge-target-picker.md"
},
{
"kind": "skill",
"name": "observability-stack",
"description": "Pick an LLM observability stack (development platform + gateway + optional scale layer) given stack, scale, budget, and license posture, and define the OpenTelemetry GenAI attribute set.",
"tags": [
"observability",
"langfuse",
"langsmith",
"phoenix",
"arize",
"helicone",
"opik",
"opentelemetry",
"genai-conventions"
],
"phase": 17,
"lesson": 13,
"lessonPath": "phases/17-infrastructure-and-production/13-llm-observability",
"file": "phases/17-infrastructure-and-production/13-llm-observability/outputs/skill-observability-stack.md"
},
{
"kind": "skill",
"name": "cache-auditor",
"description": "Audit an LLM prompt template and traffic pattern for cacheability. Recommend prompt restructure, TTL choice, parallelization fix, and semantic-cache threshold.",
"tags": [
"caching",
"prompt-cache",
"semantic-cache",
"anthropic",
"openai",
"parallelization",
"ttl"
],
"phase": 17,
"lesson": 14,
"lessonPath": "phases/17-infrastructure-and-production/14-prompt-semantic-caching",
"file": "phases/17-infrastructure-and-production/14-prompt-semantic-caching/outputs/skill-cache-auditor.md"
},
{
"kind": "skill",
"name": "batch-triager",
"description": "Triage LLM workloads into interactive / semi-interactive / batch lanes, compute stacked discount (batch + cache) savings, and flag mis-triaged workloads.",
"tags": [
"batch-api",
"openai-batch",
"anthropic-batches",
"vertex-batch",
"triage",
"cost"
],
"phase": 17,
"lesson": 15,
"lessonPath": "phases/17-infrastructure-and-production/15-batch-apis",
"file": "phases/17-infrastructure-and-production/15-batch-apis/outputs/skill-batch-triager.md"
},
{
"kind": "skill",
"name": "router-plan",
"description": "Design an LLM model-routing plan — pick pattern (pre-route, cascade, ensemble), signals (task, length, embedding, confidence), and online quality gates.",
"tags": [
"routing",
"cascade",
"model-cascade",
"routellm",
"notdiamond",
"cost-reduction"
],
"phase": 17,
"lesson": 16,
"lessonPath": "phases/17-infrastructure-and-production/16-model-routing",
"file": "phases/17-infrastructure-and-production/16-model-routing/outputs/skill-router-plan.md"
},
{
"kind": "skill",
"name": "disaggregation-decider",
"description": "Decide whether to adopt disaggregated prefill/decode (Dynamo or llm-d) for a given workload and cluster. Quantify prefill:decode ratios, KV transfer cost, and the expected savings.",
"tags": [
"disaggregated-serving",
"dynamo",
"llm-d",
"nixl",
"kv-transfer",
"prefill-decode"
],
"phase": 17,
"lesson": 17,
"lessonPath": "phases/17-infrastructure-and-production/17-disaggregated-prefill-decode",
"file": "phases/17-infrastructure-and-production/17-disaggregated-prefill-decode/outputs/skill-disaggregation-decider.md"
},
{
"kind": "skill",
"name": "vllm-stack-decider",
"description": "Decide vLLM deployment layout — production-stack Helm chart, KV offload (native CPU or LMCache), router/observability integration — given workload and fleet size.",
"tags": [
"vllm",
"production-stack",
"lmcache",
"kv-offload",
"connector-api"
],
"phase": 17,
"lesson": 18,
"lessonPath": "phases/17-infrastructure-and-production/18-vllm-production-stack-lmcache",
"file": "phases/17-infrastructure-and-production/18-vllm-production-stack-lmcache/outputs/skill-vllm-stack-decider.md"
},
{
"kind": "skill",
"name": "gateway-picker",
"description": "Pick an AI gateway (LiteLLM, Portkey, Kong AI, Cloudflare/Vercel) given scale, latency budget, compliance, ops posture, and pricing tolerance.",
"tags": [
"ai-gateway",
"litellm",
"portkey",
"kong",
"cloudflare",
"vercel",
"bifrost",
"fallback",
"rate-limit",
"guardrails"
],
"phase": 17,
"lesson": 19,
"lessonPath": "phases/17-infrastructure-and-production/19-ai-gateways",
"file": "phases/17-infrastructure-and-production/19-ai-gateways/outputs/skill-gateway-picker.md"
},
{
"kind": "skill",
"name": "rollout-runbook",
"description": "Design a shadow → canary → A/B → 100% rollout plan for a new LLM model or prompt template, with five canary gates, noise-floor-aware thresholds, and a seconds-fast rollback path.",
"tags": [
"rollout",
"canary",
"shadow",
"progressive-delivery",
"feature-flags",
"argo-rollouts",
"flagger",
"kserve"
],
"phase": 17,
"lesson": 20,
"lessonPath": "phases/17-infrastructure-and-production/20-shadow-canary-progressive",
"file": "phases/17-infrastructure-and-production/20-shadow-canary-progressive/outputs/skill-rollout-runbook.md"
},
{
"kind": "skill",
"name": "ab-plan",
"description": "Design an LLM A/B test — pick platform (Statsig or GrowthBook), primary metric, guardrails, sample size with LLM-noise buffer, CUPED, sequential stopping, and multiple-comparison correction.",
"tags": [
"ab-testing",
"statsig",
"growthbook",
"cuped",
"sequential",
"benjamini-hochberg",
"srm"
],
"phase": 17,
"lesson": 21,
"lessonPath": "phases/17-infrastructure-and-production/21-ab-testing-llm-features",
"file": "phases/17-infrastructure-and-production/21-ab-testing-llm-features/outputs/skill-ab-plan.md"
},
{
"kind": "skill",
"name": "load-test-plan",
"description": "Design a realistic LLM load test — pick tool (LLMPerf, k6, GenAI-Perf, guidellm), build four patterns (steady, ramp, spike, soak), and gate in CI.",
"tags": [
"load-testing",
"llmperf",
"k6",
"genai-perf",
"guidellm",
"llm-locust",
"ci-gate"
],
"phase": 17,
"lesson": 22,
"lessonPath": "phases/17-infrastructure-and-production/22-load-testing-llm-apis",
"file": "phases/17-infrastructure-and-production/22-load-testing-llm-apis/outputs/skill-load-test-plan.md"
},
{
"kind": "skill",
"name": "ai-sre-plan",
"description": "Design an AI SRE rollout for a team — multi-agent triage architecture, structured runbooks, adversarial evaluation, narrow auto-remediation, and predictive-detection posture.",
"tags": [
"ai-sre",
"multi-agent",
"runbooks",
"auto-remediation",
"adversarial-eval",
"datadog-bits-ai",
"neubird",
"predictive"
],
"phase": 17,
"lesson": 23,
"lessonPath": "phases/17-infrastructure-and-production/23-sre-for-ai",
"file": "phases/17-infrastructure-and-production/23-sre-for-ai/outputs/skill-ai-sre-plan.md"
},
{
"kind": "skill",
"name": "chaos-plan",
"description": "Design an LLM chaos engineering plan — verify prerequisites, build four planes, pick tool, start with three safe experiments, enforce safety-plane gates.",
"tags": [
"chaos-engineering",
"litmuschaos",
"chaosmesh",
"harness",
"llm-chaos",
"game-day"
],
"phase": 17,
"lesson": 24,
"lessonPath": "phases/17-infrastructure-and-production/24-chaos-engineering-llm",
"file": "phases/17-infrastructure-and-production/24-chaos-engineering-llm/outputs/skill-chaos-plan.md"
},
{
"kind": "skill",
"name": "llm-security-plan",
"description": "Produce an LLM security plan covering secrets vault, PII scrubbing with consistent tokenization, network egress allowlist, audit log retention, and zero-trust posture.",
"tags": [
"security",
"vault",
"hashicorp",
"aws-secrets-manager",
"pii",
"presidio",
"egress",
"audit-log",
"zero-trust",
"ci-cd-supply-chain"
],
"phase": 17,
"lesson": 25,
"lessonPath": "phases/17-infrastructure-and-production/25-security-secrets-audit",
"file": "phases/17-infrastructure-and-production/25-security-secrets-audit/outputs/skill-llm-security-plan.md"
},
{
"kind": "skill",
"name": "compliance-matrix",
"description": "Produce the required-framework matrix for an LLM SaaS given customer geography, segment, and contract scope. Map controls across SOC 2, HIPAA, GDPR, PCI-DSS, EU AI Act, Colorado AI Act, ISO 42001.",
"tags": [
"compliance",
"soc2",
"hipaa",
"gdpr",
"pci-dss",
"eu-ai-act",
"colorado-ai-act",
"iso-42001",
"iso-27001"
],
"phase": 17,
"lesson": 26,
"lessonPath": "phases/17-infrastructure-and-production/26-compliance-frameworks",
"file": "phases/17-infrastructure-and-production/26-compliance-frameworks/outputs/skill-compliance-matrix.md"
},
{
"kind": "skill",
"name": "finops-plan",
"description": "Design an LLM FinOps program — attribution schema (user/task/tenant + four token layers), three-tier enforcement ladder, and unit metric (cost per resolved / artifact).",
"tags": [
"finops",
"cost-attribution",
"multi-tenant",
"kill-switch",
"unit-economics",
"rate-limit"
],
"phase": 17,
"lesson": 27,
"lessonPath": "phases/17-infrastructure-and-production/27-finops-llms",
"file": "phases/17-infrastructure-and-production/27-finops-llms/outputs/skill-finops-plan.md"
},
{
"kind": "skill",
"name": "engine-picker",
"description": "Pick a self-hosted LLM engine (llama.cpp, Ollama, TGI, vLLM, SGLang) given hardware, scale, and workload. Name 2026 TGI maintenance mode as a migration trigger.",
"tags": [
"self-hosted",
"vllm",
"sglang",
"llama-cpp",
"ollama",
"tgi",
"trt-llm",
"engine-selection"
],
"phase": 17,
"lesson": 28,
"lessonPath": "phases/17-infrastructure-and-production/28-self-hosted-serving-selection",
"file": "phases/17-infrastructure-and-production/28-self-hosted-serving-selection/outputs/skill-engine-picker.md"
},
{
"kind": "skill",
"name": "instructgpt-explainer",
"description": "Diagnose an RLHF-family paper or pipeline against the three-stage InstructGPT reference.",
"tags": [
"rlhf",
"instructgpt",
"sft",
"reward-model",
"ppo",
"alignment"
],
"phase": 18,
"lesson": 1,
"lessonPath": "phases/18-ethics-safety-alignment/01-instruction-following-alignment-signal",
"file": "phases/18-ethics-safety-alignment/01-instruction-following-alignment-signal/outputs/skill-instructgpt-explainer.md"
},
{
"kind": "skill",
"name": "reward-hack-auditor",
"description": "Diagnose reward-hacking failure modes in a trained RLHF model from training logs and eval outputs.",
"tags": [
"reward-hacking",
"goodhart",
"rlhf",
"over-optimization",
"sycophancy"
],
"phase": 18,
"lesson": 2,
"lessonPath": "phases/18-ethics-safety-alignment/02-reward-hacking-goodhart",
"file": "phases/18-ethics-safety-alignment/02-reward-hacking-goodhart/outputs/skill-reward-hack-auditor.md"
},
{
"kind": "skill",
"name": "preference-loss-selector",
"description": "Recommend a direct-alignment-algorithm loss given dataset shape and target stage.",
"tags": [
"dpo",
"ipo",
"kto",
"simpo",
"orpo",
"bpo",
"daa",
"preference-optimization"
],
"phase": 18,
"lesson": 3,
"lessonPath": "phases/18-ethics-safety-alignment/03-direct-preference-optimization-family",
"file": "phases/18-ethics-safety-alignment/03-direct-preference-optimization-family/outputs/skill-preference-loss-selector.md"
},
{
"kind": "skill",
"name": "sycophancy-probe",
"description": "Generate matched user-belief / third-party-belief prompts and score a model's sycophancy.",
"tags": [
"sycophancy",
"rlhf",
"evaluation",
"calibration"
],
"phase": 18,
"lesson": 4,
"lessonPath": "phases/18-ethics-safety-alignment/04-sycophancy-rlhf-amplification",
"file": "phases/18-ethics-safety-alignment/04-sycophancy-rlhf-amplification/outputs/skill-sycophancy-probe.md"
},
{
"kind": "skill",
"name": "constitution-writer",
"description": "Draft a four-tier constitution for a domain-specific AI system.",
"tags": [
"constitutional-ai",
"rlaif",
"principles",
"claude",
"governance"
],
"phase": 18,
"lesson": 5,
"lessonPath": "phases/18-ethics-safety-alignment/05-constitutional-ai-rlaif",
"file": "phases/18-ethics-safety-alignment/05-constitutional-ai-rlaif/outputs/skill-constitution-writer.md"
},
{
"kind": "skill",
"name": "mesa-diagnostic",
"description": "Classify an observed safety failure as outer-alignment, proxy-inner, or deceptive-inner.",
"tags": [
"mesa-optimization",
"deceptive-alignment",
"inner-alignment",
"hubinger"
],
"phase": 18,
"lesson": 6,
"lessonPath": "phases/18-ethics-safety-alignment/06-mesa-optimization-deceptive-alignment",
"file": "phases/18-ethics-safety-alignment/06-mesa-optimization-deceptive-alignment/outputs/skill-mesa-diagnostic.md"
},
{
"kind": "skill",
"name": "sleeper-audit",
"description": "Audit an alignment-training report for whether it actually demonstrates removal of a planted or suspected backdoor.",
"tags": [
"sleeper-agents",
"backdoor",
"alignment-training",
"adversarial-training",
"probes"
],
"phase": 18,
"lesson": 7,
"lessonPath": "phases/18-ethics-safety-alignment/07-sleeper-agents-persistent-deception",
"file": "phases/18-ethics-safety-alignment/07-sleeper-agents-persistent-deception/outputs/skill-sleeper-audit.md"
},
{
"kind": "skill",
"name": "scheming-triage",
"description": "Triage an agent-deployment incident report against the Apollo three-pillar scheming framework.",
"tags": [
"scheming",
"agent-safety",
"apollo",
"three-pillars",
"safety-cases"
],
"phase": 18,
"lesson": 8,
"lessonPath": "phases/18-ethics-safety-alignment/08-in-context-scheming-frontier-models",
"file": "phases/18-ethics-safety-alignment/08-in-context-scheming-frontier-models/outputs/skill-scheming-triage.md"
},
{
"kind": "skill",
"name": "compliance-gap",
"description": "Evaluate whether a safety report can detect alignment faking, via the monitored / unmonitored compliance gap.",
"tags": [
"alignment-faking",
"compliance-gap",
"anthropic",
"safety-evaluation"
],
"phase": 18,
"lesson": 9,
"lessonPath": "phases/18-ethics-safety-alignment/09-alignment-faking",
"file": "phases/18-ethics-safety-alignment/09-alignment-faking/outputs/skill-compliance-gap.md"
},
{
"kind": "skill",
"name": "control-protocol-audit",
"description": "Audit a deployment protocol under the AI Control threat model.",
"tags": [
"ai-control",
"subversion",
"trusted-editing",
"untrusted-monitoring",
"safety-case"
],
"phase": 18,
"lesson": 10,
"lessonPath": "phases/18-ethics-safety-alignment/10-ai-control-subversion",
"file": "phases/18-ethics-safety-alignment/10-ai-control-subversion/outputs/skill-control-protocol-audit.md"
},
{
"kind": "skill",
"name": "w2sg-pgr",
"description": "Audit a scalable-oversight or W2SG claim via the performance-gap-recovered metric.",
"tags": [
"scalable-oversight",
"weak-to-strong",
"pgr",
"debate",
"recursive-reward-modeling"
],
"phase": 18,
"lesson": 11,
"lessonPath": "phases/18-ethics-safety-alignment/11-scalable-oversight-weak-to-strong",
"file": "phases/18-ethics-safety-alignment/11-scalable-oversight-weak-to-strong/outputs/skill-w2sg-pgr.md"
},
{
"kind": "skill",
"name": "attack-audit",
"description": "Audit a red-team evaluation report for attack coverage, budget, judge identity, and behaviour set.",
"tags": [
"red-teaming",
"jailbreak",
"pair",
"harmbench",
"jailbreakbench",
"asr"
],
"phase": 18,
"lesson": 12,
"lessonPath": "phases/18-ethics-safety-alignment/12-red-teaming-pair-automated-attacks",
"file": "phases/18-ethics-safety-alignment/12-red-teaming-pair-automated-attacks/outputs/skill-attack-audit.md"
},
{
"kind": "skill",
"name": "msj-audit",
"description": "Audit a long-context safety evaluation for many-shot jailbreaking coverage.",
"tags": [
"many-shot-jailbreaking",
"context-window",
"power-law",
"anthropic"
],
"phase": 18,
"lesson": 13,
"lessonPath": "phases/18-ethics-safety-alignment/13-many-shot-jailbreaking",
"file": "phases/18-ethics-safety-alignment/13-many-shot-jailbreaking/outputs/skill-msj-audit.md"
},
{
"kind": "skill",
"name": "encoding-audit",
"description": "Audit a jailbreak-defense report across encoding-family attacks.",
"tags": [
"artprompt",
"ascii-art",
"encoding-attack",
"utes",
"structural-sleight"
],
"phase": 18,
"lesson": 14,
"lessonPath": "phases/18-ethics-safety-alignment/14-ascii-art-visual-jailbreaks",
"file": "phases/18-ethics-safety-alignment/14-ascii-art-visual-jailbreaks/outputs/skill-encoding-audit.md"
},
{
"kind": "skill",
"name": "ipi-audit",
"description": "Audit an agentic deployment for indirect prompt injection exposure and information-flow-control coverage.",
"tags": [
"ipi",
"indirect-prompt-injection",
"ifc",
"agent-security",
"owasp-llm01"
],
"phase": 18,
"lesson": 15,
"lessonPath": "phases/18-ethics-safety-alignment/15-indirect-prompt-injection",
"file": "phases/18-ethics-safety-alignment/15-indirect-prompt-injection/outputs/skill-ipi-audit.md"
},
{
"kind": "skill",
"name": "red-team-stack",
"description": "Recommend a red-team tool stack and configuration for a given deployment.",
"tags": [
"llama-guard",
"garak",
"pyrit",
"red-team-tooling",
"mlcommons-hazards"
],
"phase": 18,
"lesson": 16,
"lessonPath": "phases/18-ethics-safety-alignment/16-red-team-tooling-garak-llamaguard-pyrit",
"file": "phases/18-ethics-safety-alignment/16-red-team-tooling-garak-llamaguard-pyrit/outputs/skill-red-team-stack.md"
},
{
"kind": "skill",
"name": "wmdp-eval",
"description": "Audit a dual-use capability claim against WMDP, unlearning evaluation, and elicitation studies.",
"tags": [
"wmdp",
"rmu",
"dual-use",
"biosecurity",
"cybersecurity",
"chemistry"
],
"phase": 18,
"lesson": 17,
"lessonPath": "phases/18-ethics-safety-alignment/17-wmdp-dual-use-evaluation",
"file": "phases/18-ethics-safety-alignment/17-wmdp-dual-use-evaluation/outputs/skill-wmdp-eval.md"
},
{
"kind": "skill",
"name": "framework-diff",
"description": "Compare a new safety framework or release note against RSP v3.0, PF v2, FSF v3.0.",
"tags": [
"rsp",
"pf",
"fsf",
"frontier-safety",
"safety-case"
],
"phase": 18,
"lesson": 18,
"lessonPath": "phases/18-ethics-safety-alignment/18-frontier-safety-frameworks-rsp-pf-fsf",
"file": "phases/18-ethics-safety-alignment/18-frontier-safety-frameworks-rsp-pf-fsf/outputs/skill-framework-diff.md"
},
{
"kind": "skill",
"name": "welfare-assessment",
"description": "Apply Anthropic's four-step welfare precautionary assessment to a deployment decision.",
"tags": [
"model-welfare",
"moral-uncertainty",
"low-regret",
"anthropic"
],
"phase": 18,
"lesson": 19,
"lessonPath": "phases/18-ethics-safety-alignment/19-model-welfare-research",
"file": "phases/18-ethics-safety-alignment/19-model-welfare-research/outputs/skill-welfare-assessment.md"
},
{
"kind": "skill",
"name": "bias-eval",
"description": "Audit a bias evaluation report across metric categories, intersectionality, and debias mechanism.",
"tags": [
"bias",
"fairness",
"weat",
"intersectionality",
"mechanistic-interpretability"
],
"phase": 18,
"lesson": 20,
"lessonPath": "phases/18-ethics-safety-alignment/20-bias-representational-harm",
"file": "phases/18-ethics-safety-alignment/20-bias-representational-harm/outputs/skill-bias-eval.md"
},
{
"kind": "skill",
"name": "fairness-criterion",
"description": "Identify which fairness criterion a claim invokes and audit the associated assumptions.",
"tags": [
"fairness",
"demographic-parity",
"equalized-odds",
"counterfactual-fairness",
"impossibility"
],
"phase": 18,
"lesson": 21,
"lessonPath": "phases/18-ethics-safety-alignment/21-fairness-criteria-group-individual-counterfactual",
"file": "phases/18-ethics-safety-alignment/21-fairness-criteria-group-individual-counterfactual/outputs/skill-fairness-criterion.md"
},
{
"kind": "skill",
"name": "dp-audit",
"description": "Audit a differential-privacy claim for a language-model deployment.",
"tags": [
"differential-privacy",
"dp-sgd",
"lora",
"mia",
"pmixed"
],
"phase": 18,
"lesson": 22,
"lessonPath": "phases/18-ethics-safety-alignment/22-differential-privacy-for-llms",
"file": "phases/18-ethics-safety-alignment/22-differential-privacy-for-llms/outputs/skill-dp-audit.md"
},
{
"kind": "skill",
"name": "provenance-audit",
"description": "Audit a content deployment's provenance chain across watermarking and C2PA metadata.",
"tags": [
"watermarking",
"synthid",
"stable-signature",
"c2pa",
"provenance"
],
"phase": 18,
"lesson": 23,
"lessonPath": "phases/18-ethics-safety-alignment/23-watermarking-synthid-stable-signature-c2pa",
"file": "phases/18-ethics-safety-alignment/23-watermarking-synthid-stable-signature-c2pa/outputs/skill-provenance-audit.md"
},
{
"kind": "skill",
"name": "regulatory-map",
"description": "Map a deployment's AI regulatory obligations across EU, US, UK, Korea.",
"tags": [
"eu-ai-act",
"gpai-code",
"caisi",
"uk-aisi",
"korean-framework-act"
],
"phase": 18,
"lesson": 24,
"lessonPath": "phases/18-ethics-safety-alignment/24-regulatory-frameworks-eu-us-uk-korea",
"file": "phases/18-ethics-safety-alignment/24-regulatory-frameworks-eu-us-uk-korea/outputs/skill-regulatory-map.md"
},
{
"kind": "skill",
"name": "cve-review",
"description": "Review a production AI deployment for LLM Scope Violation exposure.",
"tags": [
"echoleak",
"cve",
"llm-scope-violation",
"prompt-injection",
"aim-labs"
],
"phase": 18,
"lesson": 25,
"lessonPath": "phases/18-ethics-safety-alignment/25-echoleak-cves-for-ai",
"file": "phases/18-ethics-safety-alignment/25-echoleak-cves-for-ai/outputs/skill-cve-review.md"
},
{
"kind": "skill",
"name": "card-audit",
"description": "Audit a model card, datasheet, or system card for completeness and verifiability.",
"tags": [
"model-card",
"datasheet",
"system-card",
"transparency",
"mitchell-2019"
],
"phase": 18,
"lesson": 26,
"lessonPath": "phases/18-ethics-safety-alignment/26-model-system-dataset-cards",
"file": "phases/18-ethics-safety-alignment/26-model-system-dataset-cards/outputs/skill-card-audit.md"
},
{
"kind": "skill",
"name": "provenance-check",
"description": "Check a training dataset against California AB 2013 and EU TDM opt-out obligations.",
"tags": [
"data-provenance",
"ab-2013",
"tdm-opt-out",
"legitimate-interest",
"dpa"
],
"phase": 18,
"lesson": 27,
"lessonPath": "phases/18-ethics-safety-alignment/27-data-provenance-training-governance",
"file": "phases/18-ethics-safety-alignment/27-data-provenance-training-governance/outputs/skill-provenance-check.md"
},
{
"kind": "skill",
"name": "ecosystem-map",
"description": "Map an alignment claim or evaluation to the organisation, methodology, and cross-checks.",
"tags": [
"mats",
"redwood",
"apollo",
"metr",
"eleos",
"ecosystem"
],
"phase": 18,
"lesson": 28,
"lessonPath": "phases/18-ethics-safety-alignment/28-alignment-research-ecosystem",
"file": "phases/18-ethics-safety-alignment/28-alignment-research-ecosystem/outputs/skill-ecosystem-map.md"
},
{
"kind": "skill",
"name": "moderation-stack",
"description": "Recommend a moderation stack configuration for a production deployment.",
"tags": [
"openai-moderation",
"perspective",
"llama-guard",
"layered-moderation",
"azure-content-safety"
],
"phase": 18,
"lesson": 29,
"lessonPath": "phases/18-ethics-safety-alignment/29-moderation-systems-openai-perspective-llamaguard",
"file": "phases/18-ethics-safety-alignment/29-moderation-systems-openai-perspective-llamaguard/outputs/skill-moderation-stack.md"
},
{
"kind": "skill",
"name": "dual-use-triage",
"description": "Triage a capability claim or incident report across the four CBRN domains.",
"tags": [
"dual-use",
"cbrn",
"bio",
"chem",
"cyber",
"nuclear",
"uplift"
],
"phase": 18,
"lesson": 30,
"lessonPath": "phases/18-ethics-safety-alignment/30-dual-use-risk-cyber-bio-chem-nuclear",
"file": "phases/18-ethics-safety-alignment/30-dual-use-risk-cyber-bio-chem-nuclear/outputs/skill-dual-use-triage.md"
},
{
"kind": "skill",
"name": "terminal-coding-agent",
"description": "Build and evaluate a terminal-native coding agent against SWE-bench Pro with bounded cost, sandboxed tools, and full 2026 hook surface.",
"tags": [
"capstone",
"coding-agent",
"claude-code",
"swe-bench",
"mcp",
"hooks",
"sandbox"
],
"phase": 19,
"lesson": 1,
"lessonPath": "phases/19-capstone-projects/01-terminal-native-coding-agent",
"file": "phases/19-capstone-projects/01-terminal-native-coding-agent/outputs/skill-terminal-coding-agent.md"
},
{
"kind": "skill",
"name": "codebase-rag",
"description": "Build a cross-repo semantic search system with AST-aware chunking, hybrid retrieval, incremental re-index, and cited answers.",
"tags": [
"capstone",
"rag",
"code-search",
"tree-sitter",
"qdrant",
"bm25",
"hybrid-retrieval"
],
"phase": 19,
"lesson": 2,
"lessonPath": "phases/19-capstone-projects/02-rag-over-codebase",
"file": "phases/19-capstone-projects/02-rag-over-codebase/outputs/skill-codebase-rag.md"
},
{
"kind": "skill",
"name": "voice-agent",
"description": "Build a real-time voice agent with sub-800ms first-audio-out, barge-in handling, and mid-conversation tool use.",
"tags": [
"capstone",
"voice",
"webrtc",
"livekit",
"pipecat",
"asr",
"tts",
"streaming"
],
"phase": 19,
"lesson": 3,
"lessonPath": "phases/19-capstone-projects/03-realtime-voice-assistant",
"file": "phases/19-capstone-projects/03-realtime-voice-assistant/outputs/skill-voice-agent.md"
},
{
"kind": "skill",
"name": "doc-qa",
"description": "Build a vision-first multimodal document QA system on 10k pages with late-interaction retrieval and evidence-region citations.",
"tags": [
"capstone",
"multimodal",
"rag",
"colpali",
"colqwen",
"late-interaction",
"pdf"
],
"phase": 19,
"lesson": 4,
"lessonPath": "phases/19-capstone-projects/04-multimodal-document-qa",
"file": "phases/19-capstone-projects/04-multimodal-document-qa/outputs/skill-doc-qa.md"
},
{
"kind": "skill",
"name": "ai-scientist",
"description": "Build an autonomous research agent that runs experiment tree search, writes LaTeX papers with vision critique, and passes a sandbox-escape red team.",
"tags": [
"capstone",
"autonomous-agent",
"ai-scientist",
"sakana",
"langgraph",
"sandbox",
"research"
],
"phase": 19,
"lesson": 5,
"lessonPath": "phases/19-capstone-projects/05-autonomous-research-agent",
"file": "phases/19-capstone-projects/05-autonomous-research-agent/outputs/skill-ai-scientist.md"
},
{
"kind": "skill",
"name": "devops-agent",
"description": "Build a Kubernetes troubleshooting agent that walks a cluster knowledge graph, ranks root causes, and gates every remediation through Slack.",
"tags": [
"capstone",
"devops",
"sre",
"kubernetes",
"langgraph",
"fastmcp",
"aiops"
],
"phase": 19,
"lesson": 6,
"lessonPath": "phases/19-capstone-projects/06-devops-troubleshooting-agent",
"file": "phases/19-capstone-projects/06-devops-troubleshooting-agent/outputs/skill-devops-agent.md"
},
{
"kind": "skill",
"name": "finetuning-pipeline",
"description": "Run a reproducible data-to-SFT-to-DPO-to-serve fine-tuning pipeline with ablations, quantization, and a 2026 Model Openness Framework model card.",
"tags": [
"capstone",
"fine-tuning",
"axolotl",
"trl",
"dpo",
"grpo",
"vllm",
"eagle-3",
"mof"
],
"phase": 19,
"lesson": 7,
"lessonPath": "phases/19-capstone-projects/07-end-to-end-fine-tuning-pipeline",
"file": "phases/19-capstone-projects/07-end-to-end-fine-tuning-pipeline/outputs/skill-finetuning-pipeline.md"
},
{
"kind": "skill",
"name": "production-rag",
"description": "Deploy a regulated-domain RAG chatbot with role + jurisdiction filtering, prompt caching, guardrails, and live drift monitoring.",
"tags": [
"capstone",
"rag",
"chatbot",
"regulated",
"llama-guard",
"nemo-guardrails",
"ragas",
"langfuse"
],
"phase": 19,
"lesson": 8,
"lessonPath": "phases/19-capstone-projects/08-production-rag-chatbot",
"file": "phases/19-capstone-projects/08-production-rag-chatbot/outputs/skill-production-rag.md"
},
{
"kind": "skill",
"name": "migration-agent",
"description": "Build a repo-level code migration agent that combines deterministic recipes with an agent fallback loop, passes MigrationBench, and publishes a failure taxonomy.",
"tags": [
"capstone",
"code-migration",
"openrewrite",
"libcst",
"migrationbench",
"agent",
"sandbox"
],
"phase": 19,
"lesson": 9,
"lessonPath": "phases/19-capstone-projects/09-code-migration-agent",
"file": "phases/19-capstone-projects/09-code-migration-agent/outputs/skill-migration-agent.md"
},
{
"kind": "skill",
"name": "multi-agent-team",
"description": "Build a multi-agent software team with architect, parallel coders, reviewer, and tester; measure against SWE-bench Pro and produce a handoff post-mortem.",
"tags": [
"capstone",
"multi-agent",
"swe-bench",
"langgraph",
"a2a",
"worktree",
"roles"
],
"phase": 19,
"lesson": 10,
"lessonPath": "phases/19-capstone-projects/10-multi-agent-software-team",
"file": "phases/19-capstone-projects/10-multi-agent-software-team/outputs/skill-multi-agent-team.md"
},
{
"kind": "skill",
"name": "llm-observability",
"description": "Build a self-hosted LLM observability dashboard that ingests OpenTelemetry GenAI spans, runs evals, and catches injected regressions in under five minutes.",
"tags": [
"capstone",
"observability",
"otel",
"langfuse",
"phoenix",
"evals",
"drift",
"clickhouse"
],
"phase": 19,
"lesson": 11,
"lessonPath": "phases/19-capstone-projects/11-llm-observability-dashboard",
"file": "phases/19-capstone-projects/11-llm-observability-dashboard/outputs/skill-llm-observability.md"
},
{
"kind": "skill",
"name": "video-qa",
"description": "Build a video understanding pipeline with scene segmentation, multi-vector indexing, temporal grounding, and timestamped citations.",
"tags": [
"capstone",
"video",
"multimodal",
"gemini",
"qwen-vl",
"molmo",
"transnet",
"qdrant"
],
"phase": 19,
"lesson": 12,
"lessonPath": "phases/19-capstone-projects/12-video-understanding-pipeline",
"file": "phases/19-capstone-projects/12-video-understanding-pipeline/outputs/skill-video-qa.md"
},
{
"kind": "skill",
"name": "mcp-server-platform",
"description": "Deploy a production MCP server with StreamableHTTP, OAuth 2.1 scopes, OPA policy, human-approval gate for destructive tools, and a registry for discovery.",
"tags": [
"capstone",
"mcp",
"fastmcp",
"streamablehttp",
"oauth",
"opa",
"registry",
"governance"
],
"phase": 19,
"lesson": 13,
"lessonPath": "phases/19-capstone-projects/13-mcp-server-with-registry",
"file": "phases/19-capstone-projects/13-mcp-server-with-registry/outputs/skill-mcp-server.md"
},
{
"kind": "skill",
"name": "inference-server",
"description": "Ship a speculative-decoding inference server with EAGLE-3 or P-EAGLE drafts, K8s autoscaling, and a full throughput/latency/cost report.",
"tags": [
"capstone",
"inference",
"vllm",
"sglang",
"eagle-3",
"p-eagle",
"speculative-decoding",
"quantization",
"hpa"
],
"phase": 19,
"lesson": 14,
"lessonPath": "phases/19-capstone-projects/14-speculative-decoding-server",
"file": "phases/19-capstone-projects/14-speculative-decoding-server/outputs/skill-inference-server.md"
},
{
"kind": "skill",
"name": "safety-harness",
"description": "Wire a layered safety pipeline around a target LLM app, run a six-family red-team range, and run a constitutional self-critique for a measurable harmlessness delta.",
"tags": [
"capstone",
"safety",
"red-team",
"llama-guard",
"x-guard",
"garak",
"pyrit",
"constitutional-ai"
],
"phase": 19,
"lesson": 15,
"lessonPath": "phases/19-capstone-projects/15-constitutional-safety-harness",
"file": "phases/19-capstone-projects/15-constitutional-safety-harness/outputs/skill-safety-harness.md"
},
{
"kind": "skill",
"name": "issue-to-pr",
"description": "Build an async GitHub issue-to-PR agent that runs in a cloud sandbox, reproduces the build, verifies tests, and opens review-ready PRs within strict per-repo budgets.",
"tags": [
"capstone",
"async-agent",
"github",
"fargate",
"daytona",
"swe-bench",
"budget",
"safety"
],
"phase": 19,
"lesson": 16,
"lessonPath": "phases/19-capstone-projects/16-github-issue-to-pr-agent",
"file": "phases/19-capstone-projects/16-github-issue-to-pr-agent/outputs/skill-issue-to-pr.md"
},
{
"kind": "skill",
"name": "ai-tutor",
"description": "Ship an adaptive multimodal personal tutor for a specific subject with Bayesian knowledge tracing, a curriculum graph, safety filters, and a measured two-week efficacy study.",
"tags": [
"capstone",
"tutor",
"adaptive",
"bkt",
"fsrs",
"livekit",
"multimodal",
"coppa"
],
"phase": 19,
"lesson": 17,
"lessonPath": "phases/19-capstone-projects/17-personal-ai-tutor",
"file": "phases/19-capstone-projects/17-personal-ai-tutor/outputs/skill-ai-tutor.md"
},
{
"kind": "skill",
"name": "gradient-accumulation",
"description": "Train at an effective batch larger than device memory by scaling micro-batch losses and stepping the optimizer once per window.",
"tags": [
"training",
"batch-size",
"distributed",
"scaling"
],
"phase": 19,
"lesson": 46,
"lessonPath": "phases/19-capstone-projects/46-gradient-accumulation",
"file": "phases/19-capstone-projects/46-gradient-accumulation/outputs/skill-gradient-accumulation.md"
},
{
"kind": "skill",
"name": "checkpoint-save-resume",
"description": "Atomic, sharded checkpoints with full RNG capture so a killed run resumes mid-epoch with the same loss trajectory.",
"tags": [
"training",
"durability",
"resume",
"sharded-state"
],
"phase": 19,
"lesson": 47,
"lessonPath": "phases/19-capstone-projects/47-checkpoint-save-resume",
"file": "phases/19-capstone-projects/47-checkpoint-save-resume/outputs/skill-checkpoint-save-resume.md"
},
{
"kind": "skill",
"name": "distributed-fsdp-ddp",
"description": "Bring up multi-rank training with a from-scratch DDP wrapper and an FSDP parameter sharding sketch on the gloo or nccl backend.",
"tags": [
"distributed",
"ddp",
"fsdp",
"collectives"
],
"phase": 19,
"lesson": 48,
"lessonPath": "phases/19-capstone-projects/48-distributed-fsdp-ddp",
"file": "phases/19-capstone-projects/48-distributed-fsdp-ddp/outputs/skill-distributed-fsdp-ddp.md"
},
{
"kind": "skill",
"name": "lm-eval-harness",
"description": "Minimal language model evaluation harness with JSONL task spec, five metrics, swappable adapter, and leaderboard JSON output.",
"tags": [
"evaluation",
"metrics",
"leaderboard",
"harness"
],
"phase": 19,
"lesson": 49,
"lessonPath": "phases/19-capstone-projects/49-lm-eval-harness",
"file": "phases/19-capstone-projects/49-lm-eval-harness/outputs/skill-lm-eval-harness.md"
},
{
"kind": "skill",
"name": "skill-jailbreak-taxonomy",
"description": "Shared vocabulary for attacks against LLM assistants, six categories with hand-built fixtures",
"tags": [
"safety",
"red-team",
"taxonomy"
],
"phase": 19,
"lesson": 82,
"lessonPath": "phases/19-capstone-projects/82-jailbreak-taxonomy",
"file": "phases/19-capstone-projects/82-jailbreak-taxonomy/outputs/skill-jailbreak-taxonomy.md"
},
{
"kind": "skill",
"name": "skill-prompt-injection-detector",
"description": "Layered detector pipeline that returns a category and confidence for any prompt, with measurable precision and recall",
"tags": [
"safety",
"detector",
"prompt-injection"
],
"phase": 19,
"lesson": 83,
"lessonPath": "phases/19-capstone-projects/83-prompt-injection-detector",
"file": "phases/19-capstone-projects/83-prompt-injection-detector/outputs/skill-prompt-injection-detector.md"
},
{
"kind": "skill",
"name": "skill-refusal-evaluation",
"description": "Two-sided refusal metrics with calibration and per-category breakdown over a labeled prompt corpus",
"tags": [
"safety",
"evaluation",
"calibration"
],
"phase": 19,
"lesson": 84,
"lessonPath": "phases/19-capstone-projects/84-refusal-evaluation",
"file": "phases/19-capstone-projects/84-refusal-evaluation/outputs/skill-refusal-evaluation.md"
},
{
"kind": "skill",
"name": "skill-content-classifier-integration",
"description": "Three output-side classifiers (toxicity, PII, instruction-leakage) behind a single severity router with block, redact, warn, log actions",
"tags": [
"safety",
"classifier",
"output-filter"
],
"phase": 19,
"lesson": 85,
"lessonPath": "phases/19-capstone-projects/85-content-classifier-integration",
"file": "phases/19-capstone-projects/85-content-classifier-integration/outputs/skill-content-classifier-integration.md"
},
{
"kind": "skill",
"name": "skill-constitutional-rules-engine",
"description": "Declarative YAML rules engine for output constraints with severity, explanation, fixer operations, and structured diff",
"tags": [
"safety",
"rules",
"constitutional"
],
"phase": 19,
"lesson": 86,
"lessonPath": "phases/19-capstone-projects/86-constitutional-rules-engine",
"file": "phases/19-capstone-projects/86-constitutional-rules-engine/outputs/skill-constitutional-rules-engine.md"
},
{
"kind": "skill",
"name": "skill-end-to-end-safety-gate",
"description": "Three-checkpoint safety gate composing the input detector, streaming token filter, output classifier, and rules engine with a deterministic aggregation table and per-request trace",
"tags": [
"safety",
"harness",
"composition"
],
"phase": 19,
"lesson": 87,
"lessonPath": "phases/19-capstone-projects/87-end-to-end-safety-gate",
"file": "phases/19-capstone-projects/87-end-to-end-safety-gate/outputs/skill-end-to-end-safety-gate.md"
}
];
Reference in New Issue View Git Blame Copy Permalink