unslothai--unsloth/.github/workflows/notebooks-ci.yml

# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.
#
# Cross-repo notebook validator. Lives in unslothai/unsloth (this repo)
# and inspects every notebook in unslothai/notebooks at HEAD (or the
# ref dispatched in via repository_dispatch).
#
# Catches the bug classes that landed in:
#   - unslothai/notebooks#258  Colab torchao 0.10 vs peft 0.19 floor
#   - unslothai/notebooks#260  DONT_UPDATE_EXCEPTIONS coverage drift
#   - unslothai/notebooks#261  torch/torchcodec ABI; --no-deps tokenizers
#   - unslothai/notebooks#264  --no-deps transformers + Colab tokenizers drift
#   - unslothai/notebooks#221  git+ HEAD installs in install cells
#   - unslothai/notebooks  commit 51b1462  template/notebook drift
#
# CPU-only by design. Layer 2 (api-introspect) reuses the existing
# tests/_zoo_aggressive_cuda_spoof.py harness so `import unsloth`
# succeeds on a GPU-less ubuntu-latest runner.

name: Notebooks CI

on:
  pull_request:
    paths:
      - 'unsloth/**'
      - 'scripts/notebook_validator.py'
      - 'scripts/notebook_to_python.py'
      - 'scripts/data/colab_pip_freeze.gpu.txt'
      - 'scripts/data/colab_to_cpu_pin.json'
      - 'tests/notebooks/**'
      - 'tests/_zoo_aggressive_cuda_spoof.py'
      - '.github/workflows/notebooks-ci.yml'
  schedule:
    # Daily 06:17 UTC. Catches Colab preinstall bumps (the upstream image
    # is rebuilt roughly weekly) without us waiting on a PR. Off the
    # :00/:30 fleet-collision spots.
    - cron: '17 6 * * *'
  workflow_dispatch:
    inputs:
      notebooks_ref:
        description: 'unslothai/notebooks ref to lint (branch / SHA / tag)'
        default: 'main'
      include_smoke:
        description: 'Also run the install-cell smoke matrix (longer)'
        type: boolean
        default: false
  repository_dispatch:
    # Fired by a tiny companion workflow on unslothai/notebooks.
    types: [notebooks_pr_opened, notebooks_main_pushed]

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

env:
  NOTEBOOKS_REF: >-
    ${{ github.event.inputs.notebooks_ref ||
        github.event.client_payload.ref ||
        'main' }}

jobs:
  static:
    name: static (drift + lint + exceptions)
    runs-on: ubuntu-latest
    timeout-minutes: 10
    steps:
      # Validate the dispatched ref before it reaches actions/checkout's `ref:`
      # input. Reading via env (NOT direct ${{ ... }} interpolation in the
      # regex test) closes the GitHub-Actions-injection class where a
      # client_payload.ref like `main"; rm -rf / #` would be embedded into the
      # shell command. NOTEBOOKS_REF defaults to 'main' on non-dispatch
      # events, but only repository_dispatch can supply attacker-controlled
      # values, so we gate this check on that event type.
      - name: Validate client_payload.ref shape
        if: github.event_name == 'repository_dispatch'
        env:
          NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
        run: |
          if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
            echo "::error::client_payload.ref contains disallowed characters" >&2
            exit 1
          fi

      - name: Checkout unsloth (this PR)
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          path: unsloth
          persist-credentials: false

      - name: Checkout unslothai/notebooks @ ${{ env.NOTEBOOKS_REF }}
        uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          repository: unslothai/notebooks
          ref: ${{ env.NOTEBOOKS_REF }}
          path: notebooks
          fetch-depth: 0  # drift check needs git status / diff
          persist-credentials: false

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: '3.12'
          cache: 'pip'

      - name: Install validator deps
        run: |
          python -m pip install --upgrade pip
          # nbformat + nbconvert come from the converter's requirements;
          # spellchecker + huggingface_hub are imported at module top of
          # update_all_notebooks.py.
          pip install \
            'nbformat>=5.10' 'nbconvert>=7.16' 'pyspellchecker>=0.8' \
            'huggingface_hub>=0.34' 'tqdm>=4.66'

      - name: Refresh Colab pip-freeze (best-effort; falls back to snapshot)
        run: |
          python unsloth/scripts/notebook_validator.py refresh-colab \
              --out unsloth/scripts/data/colab_pip_freeze.gpu.txt \
            || echo "::warning::refresh-colab failed; using committed snapshot"

      - name: Diff Colab oracle vs committed snapshots (advisory)
        # Pulls pip-freeze.gpu.txt + apt-list-gpu.txt + os-info-gpu.txt
        # from googlecolab/backend-info and prints NEW / REMOVED /
        # CHANGED entries against scripts/data/colab_*.txt. Non-blocking
        # on PRs; the daily cron job below runs the same step with
        # --strict so upstream rotations surface within ~24h.
        continue-on-error: true
        working-directory: ${{ github.workspace }}
        run: |
          python unsloth/scripts/notebook_validator.py colab-diff \
              --snapshot-dir unsloth/scripts/data

      - name: Drift check (re-run update_all_notebooks.py + git diff)
        working-directory: ${{ github.workspace }}
        # Reported as non-blocking until the upstream `unslothai/notebooks`
        # tree is regenerated. The first run on @main surfaces ~463 files
        # of drift (7359 / 9634 line delta), which is a real backlog the
        # notebooks-side maintainers need to clear in their own repo --
        # this PR's role is to surface the count, not auto-fix it.
        continue-on-error: true
        run: |
          python unsloth/scripts/notebook_validator.py drift \
              --notebooks-dir notebooks

      - name: Convert sanity (every nb / kaggle / original_template -> .py)
        # Same rationale as Drift: a handful of upstream notebooks fail
        # the converter (custom magics, malformed JSON, etc). Surface
        # the count without blocking; the team triages in unslothai/notebooks.
        continue-on-error: true
        run: |
          python unsloth/scripts/notebook_validator.py convert \
              --notebooks-dir notebooks \
              --out _converted

      - name: Lint (install cells + AST scan, env-scoped)
        # Reported as non-blocking (continue-on-error: true) until the
        # backlog of pre-existing findings on unslothai/notebooks@main is
        # cleared. Same pattern PR #5298 used for biome:check on the
        # frontend. As of this commit the live tree surfaces 27 errors +
        # 6 warnings, all real (peft/torchao floor missing in 6 nb/
        # notebooks, 14 git+ HEAD installs in hand-tuned exception
        # notebooks, 6 torch/torchcodec ABI mismatches, 1
        # transformers/tokenizers --no-deps drift). The count surfaces
        # in the PR check UI. Drop continue-on-error once it hits zero.
        continue-on-error: true
        run: |
          python unsloth/scripts/notebook_validator.py lint \
              --notebooks-dir notebooks \
              --colab-pin unsloth/scripts/data/colab_pip_freeze.gpu.txt \
              --no-pypi
        # --no-pypi skips R-INST-002 (transitive resolve via PyPI metadata).
        # Layer 1 keeps PR-time wall-clock predictable; the daily cron run
        # below drops --no-pypi and refreshes the cache.

      - name: DONT_UPDATE_EXCEPTIONS coverage
        run: |
          python unsloth/scripts/notebook_validator.py exceptions \
              --notebooks-dir notebooks

  static-with-pypi:
    name: static + transitive resolve (cron / dispatch only)
    if: ${{ github.event_name == 'schedule' || github.event_name == 'workflow_dispatch' }}
    runs-on: ubuntu-latest
    timeout-minutes: 15
    steps:
      # See `static.Validate client_payload.ref shape` for rationale. This
      # job's `if:` excludes repository_dispatch today, so the validation
      # step is a defence-in-depth no-op until that gate ever relaxes.
      - name: Validate client_payload.ref shape
        if: github.event_name == 'repository_dispatch'
        env:
          NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
        run: |
          if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
            echo "::error::client_payload.ref contains disallowed characters" >&2
            exit 1
          fi
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
          path: unsloth
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          repository: unslothai/notebooks
          ref: ${{ env.NOTEBOOKS_REF }}
          path: notebooks
          persist-credentials: false
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with: { python-version: '3.12', cache: 'pip' }
      - name: Install
        run: pip install -U pip
      - name: Refresh Colab oracle
        run: |
          python unsloth/scripts/notebook_validator.py refresh-colab \
              --out unsloth/scripts/data/colab_pip_freeze.gpu.txt
      - name: Diff Colab oracle vs committed snapshots (--strict on cron)
        # Cron-only escalation of the advisory PR-time check. Fails if
        # any of pip-freeze.gpu.txt / apt-list-gpu.txt / os-info-gpu.txt
        # has drifted from scripts/data/colab_*.txt; refresh the
        # snapshots in this repo to acknowledge.
        run: |
          python unsloth/scripts/notebook_validator.py colab-diff \
              --snapshot-dir unsloth/scripts/data --strict
      - name: Lint with live PyPI metadata
        run: |
          python unsloth/scripts/notebook_validator.py lint \
              --notebooks-dir notebooks \
              --colab-pin unsloth/scripts/data/colab_pip_freeze.gpu.txt

  api-introspect:
    name: api surface (under CUDA spoof)
    runs-on: ubuntu-latest
    timeout-minutes: 12
    steps:
      - name: Validate client_payload.ref shape
        if: github.event_name == 'repository_dispatch'
        env:
          NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
        run: |
          if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
            echo "::error::client_payload.ref contains disallowed characters" >&2
            exit 1
          fi
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
          path: unsloth
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          repository: unslothai/notebooks
          ref: ${{ env.NOTEBOOKS_REF }}
          path: notebooks
          persist-credentials: false
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with: { python-version: '3.12', cache: 'pip' }

      - name: Install CPU torch + pinned unsloth + trl + converter deps
        run: |
          python -m pip install --upgrade pip
          # CPU torch + torchvision. torchvision is required because
          # unsloth_zoo.vision_utils imports PIL at module top, and the
          # easiest way to get a torch-compatible PIL on a CPU runner is
          # to let torchvision pull the right Pillow version.
          pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
                      'torch>=2.8,<2.11' 'torchvision<0.26'
          # Pin to the same versions update_all_notebooks.py installs in
          # generated notebooks. Keep these in lockstep with PIN_TRL /
          # PIN_TRANSFORMERS in unslothai/notebooks/update_all_notebooks.py.
          # `triton` is added because unsloth/_gpu_init.py:232 does an
          # unconditional `import triton`; the PyPI wheel installs cleanly
          # on Linux x86_64 even without CUDA (same rationale as
          # consolidated-tests-ci.yml line 192-205).
          # Pillow is listed explicitly as a defensive belt-and-braces
          # next to torchvision (vision_utils crashes ModuleNotFoundError
          # if torchvision skipped its Pillow dep for any reason).
          pip install 'transformers>=4.56,<5.6' 'trl>=0.22,<0.26' 'accelerate>=1.0' \
                      'datasets>=3.4,<5' 'peft>=0.15,<0.20' \
                      'bitsandbytes>=0.43' 'sentencepiece' 'protobuf' triton \
                      Pillow safetensors tqdm packaging psutil
          # Converter deps (nbformat for notebook_to_python.py).
          pip install 'nbformat>=5.10' 'nbconvert>=7.16'
          # Install unsloth from the LOCAL checkout (the PR head), not PyPI.
          # The PR-time CI must validate the code in this PR; PyPI unsloth
          # may lag the in-repo CPU-torch fallback in unsloth/kernels/utils.py
          # (lines 162-170) that handles missing torch._C._cuda_getCurrentRawStream.
          # unsloth_zoo from git main mirrors every other CI (Core / MLX /
          # install.sh) so PR-time validation sees the same zoo HEAD.
          for attempt in 1 2 3; do
            if pip install --no-deps "unsloth_zoo @ git+https://github.com/unslothai/unsloth-zoo"; then
              break
            fi
            [ "$attempt" -eq 3 ] && { echo "::error::unsloth_zoo install failed after 3 attempts"; exit 1; }
            sleep $((5 * attempt))
          done
          pip install --no-deps -e ./unsloth

      - name: Convert notebooks for AST scan
        # Same upstream-conversion-error tolerance as the static job.
        continue-on-error: true
        run: |
          python unsloth/scripts/notebook_validator.py convert \
              --notebooks-dir notebooks --out _converted

      - name: Dump unsloth + trl API surface (under CUDA spoof)
        run: |
          PYTHONPATH=unsloth/tests python -u - <<'PY'
          import sys, json, inspect
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()
          import unsloth
          import trl
          surface = {}
          for cls_name in ("FastLanguageModel", "FastVisionModel", "FastModel"):
              cls = getattr(unsloth, cls_name, None)
              if cls is None:
                  continue
              surface[cls_name] = sorted(n for n in dir(cls) if not n.startswith("_"))
          surface["SFTConfig_kwargs"] = sorted(inspect.signature(trl.SFTConfig.__init__).parameters)
          json.dump(surface, open("_api_surface.json", "w"), indent=2)
          print("dumped surface for:", list(surface))
          PY

      - name: Run API rule against converted notebooks
        run: |
          python unsloth/scripts/notebook_validator.py api \
              --converted-dir _converted \
              --surface _api_surface.json

  smoke-install:
    name: smoke install (Colab-shaped venv, opt-in)
    if: ${{ github.event.inputs.include_smoke == 'true' || github.event_name == 'schedule' }}
    runs-on: ubuntu-latest
    timeout-minutes: 25
    strategy:
      fail-fast: false
      matrix:
        # One representative notebook per installation_*_content template.
        # Add rows when a new install template lands in update_all_notebooks.py.
        notebook:
          - 'nb/Llama3.1_(8B)-Alpaca.ipynb'           # installation_content
          - 'nb/Gemma3_(4B)-Vision.ipynb'             # installation_content + vision
          - 'nb/Llama3.1_(8B)-GRPO.ipynb'             # installation_extra_grpo_content
          - 'nb/gpt-oss-(20B)-Fine-tuning.ipynb'      # installation_gpt_oss_content
          - 'nb/Qwen3_5_(4B)_Vision.ipynb'            # installation_qwen3_5_content
          - 'nb/Nemotron-3-Nano-30B-A3B_A100.ipynb'   # installation_nemotron_nano_content
          - 'nb/Whisper.ipynb'                         # installation_whisper_content
          - 'nb/Synthetic_Data_Hackathon.ipynb'        # installation_synthetic_data_content
    steps:
      - name: Validate client_payload.ref shape
        if: github.event_name == 'repository_dispatch'
        env:
          NOTEBOOKS_REF: ${{ github.event.client_payload.ref }}
        run: |
          if ! printf '%s' "$NOTEBOOKS_REF" | grep -Eq '^[A-Za-z0-9._/-]+$'; then
            echo "::error::client_payload.ref contains disallowed characters" >&2
            exit 1
          fi
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false
          path: unsloth
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          repository: unslothai/notebooks
          ref: ${{ env.NOTEBOOKS_REF }}
          path: notebooks
          persist-credentials: false
      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with: { python-version: '3.12' }

      - name: Seed Colab-shaped venv from pip-freeze (CPU-mapped)
        run: |
          # Strip cu128 local versions, route torch/torchvision to the CPU
          # wheel index, drop CUDA-specific deps the runner can't use.
          python -u - <<'PY' > /tmp/seed_pins.txt
          import json, re
          mapping = json.load(open("unsloth/scripts/data/colab_to_cpu_pin.json"))
          rewrite = mapping["rewrite"]
          skip = set(mapping["skip"])
          spoof = set(mapping["module_spoof"])
          out = []
          for line in open("unsloth/scripts/data/colab_pip_freeze.gpu.txt"):
              line = line.strip()
              if not line or line.startswith("#"):
                  continue
              m = re.match(r"^([A-Za-z0-9._-]+)\s*==\s*(.+)$", line)
              if not m:
                  continue
              name, ver = m.group(1).lower(), m.group(2)
              if name in skip:
                  continue
              if name in spoof:
                  continue
              if name in rewrite:
                  ver = re.sub(r"[+\-].+$", "", ver)
                  out.append(f"{name}=={ver}")
              else:
                  ver = re.sub(r"[+\-].+$", "", ver)
                  out.append(f"{name}=={ver}")
          print("\n".join(out))
          PY
          head -5 /tmp/seed_pins.txt
          wc -l /tmp/seed_pins.txt

      - name: Install Colab-shaped venv
        run: |
          python -m pip install --upgrade pip
          # Best-effort: any single line that fails to resolve on CPU is
          # tolerated; the smoke contract is "the install cell + the unsloth
          # import works", not "the entire Colab venv reproduces."
          while IFS= read -r spec; do
            pip install "$spec" --index-url https://download.pytorch.org/whl/cpu \
              --extra-index-url https://pypi.org/simple || \
              echo "::warning::pin failed: $spec"
          done < /tmp/seed_pins.txt

      - name: Run install cell
        run: |
          python unsloth/scripts/notebook_validator.py convert \
              --notebooks-dir notebooks --out _converted
          # Take the converted .py and run the install cell only.
          BASE="$(basename '${{ matrix.notebook }}' .ipynb | tr -d '()' | tr -c '[:alnum:]_' _)"
          PY="_converted/${BASE}.py"
          [ -f "$PY" ] || { echo "::error::$PY not found"; ls _converted | head; exit 1; }
          # Truncate at the first `from unsloth import` so we run install +
          # core imports only.
          awk '/^from unsloth import/ { print "import sys; sys.exit(0)"; exit } { print }' "$PY" > _smoke.py
          PYTHONPATH=unsloth/tests python -u - <<'PY'
          import _zoo_aggressive_cuda_spoof as _s; _s.apply()
          # Stub torchcodec for cells that import it — no CPU wheel exists.
          import sys, types
          if "torchcodec" not in sys.modules:
              sys.modules["torchcodec"] = types.ModuleType("torchcodec")
          exec(open("_smoke.py").read(), {"__name__": "__main__"})
          PY

      - name: Verify imports under spoof
        run: |
          PYTHONPATH=unsloth/tests python -u - <<'PY'
          import sys, types
          if "torchcodec" not in sys.modules:
              sys.modules["torchcodec"] = types.ModuleType("torchcodec")
          import _zoo_aggressive_cuda_spoof as _s; _s.apply()
          import unsloth, peft, torch, torchao, transformers, tokenizers
          print("OK: imports pass under CUDA spoof")
          PY