unslothai--unsloth/.github/workflows/consolidated-tests-ci.yml

# SPDX-License-Identifier: AGPL-3.0-only
# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved.

# One consolidated CPU-only job that runs every test_* function the existing
# CI does not already cover from this repo plus the full unsloth_zoo@main
# CPU test suite plus unsloth_zoo.compiler.test_apply_fused_lm_head.
#
# Why a separate workflow:
#   - studio-backend-ci.yml's "Repo tests (CPU)" job already auto-discovers
#     tests/ minus tests/qlora, tests/saving, tests/utils, tests/sh. The 16
#     Bucket-A tests below live inside those --ignore dirs (CPU-runnable but
#     historically excluded with their GPU siblings); pulling them out into
#     a sibling job keeps the existing 760-passed baseline stable while we
#     prove the new pieces are green.
#   - unsloth_zoo has no CI on main today (.github/workflows/ is empty
#     upstream as of HEAD 030e4ba). 106 of its 111 test_* functions are
#     CPU-runnable; the 5 GPU/vLLM ones are deselected here.
#   - test_apply_fused_lm_head lives at unsloth_zoo/compiler.py:1983, not
#     under tests/, so it is not picked up by `pytest tests/`. It is a
#     plain function with no fixtures: pure regex over transformers source
#     strings, ~5-15 s wall, no GPU.
#
# Strict mode: every test step is gating (no `continue-on-error`). The
# upstream patch fixes that previously caused per-cell red have landed:
#   - unslothai/unsloth#5319 (patch_fast_lora import, patch_sft_trainer
#     Union, openenv OSError graceful skip).
#   - unslothai/unsloth-zoo#628 (MoE coverage canary so old transformers
#     skips legitimately while real discovery regressions still fail).
# After those merges every observed cell failure was one of these two
# things; if they regress we want a red cell, not a green-with-fail-prints
# cell.

name: Core

on:
  pull_request:
    paths:
      - 'unsloth/**'
      - 'unsloth_cli/**'
      - 'studio/**'
      - 'tests/**'
      - 'pyproject.toml'
      - '.github/workflows/consolidated-tests-ci.yml'
  push:
    branches: [main, pip]
  workflow_dispatch:
    inputs:
      unsloth_zoo_ref:
        description: 'unsloth_zoo git ref to test against (default main)'
        required: false
        default: 'main'

concurrency:
  group: ${{ github.workflow }}-${{ github.ref }}
  cancel-in-progress: true

permissions:
  contents: read

jobs:
  consolidated:
    # Matrix: three (transformers, TRL) combos cover the failure surface the
    # PR cares about:
    #   1. transformers==4.57.6 + TRL latest <1.0.0 (the just-before-5.x line)
    #   2. transformers latest 5.x + TRL latest 1.x (the absolute upstream tip;
    #      currently 5.8.0 + 1.3.0, both BEYOND the unsloth/unsloth_zoo
    #      <=5.5.0 / <=0.24.0 caps -- the cell exists explicitly to surface
    #      drift signal)
    #   3. transformers + TRL pinned by pyproject.toml's dependency entries
    #      (resolved dynamically at job time via tomllib)
    # fail-fast: false so each cell runs independently and a transformers /
    # TRL drift signal in one cell does not cancel the others. No
    # job-level or per-step `continue-on-error` -- real test failures now
    # fail the cell. Patches with legitimate CPU-runner preconditions
    # (real CUDA dispatcher, runtime args) are explicitly skipped via
    # NEEDS_PRECONDITION in the runtime check shim below.
    strategy:
      fail-fast: false
      matrix:
        combo:
          - id: t4576-trl0latest
            label: "HF=4.57.6 + TRL<1"
            transformers_spec: "transformers==4.57.6"
            trl_spec: "trl>=0.18.2,<1.0.0"
          - id: tlatest5-trl1latest
            label: "HF=latest + TRL=latest"
            transformers_spec: "transformers>=5,<6"
            trl_spec: "trl>=1,<2"
          - id: pyproject
            label: "HF=default + TRL=default"
            transformers_spec: "__from_pyproject__"
            trl_spec: "__from_pyproject__"
    name: "Core (${{ matrix.combo.label }})"
    runs-on: ubuntu-latest
    timeout-minutes: 35
    # No job-level or per-step `continue-on-error`. Earlier iterations
    # masked real test failures behind green check icons; that lie is
    # gone. A failing test step fails the cell. NEEDS_PRECONDITION in
    # the runtime check shim handles patches that legitimately cannot
    # run on a CPU-only runner (real CUDA dispatcher, runtime args).
    env:
      UNSLOTH_ZOO_REF: ${{ inputs.unsloth_zoo_ref || 'main' }}
      MATRIX_TRANSFORMERS_SPEC: ${{ matrix.combo.transformers_spec }}
      MATRIX_TRL_SPEC: ${{ matrix.combo.trl_spec }}
      MATRIX_COMBO_ID: ${{ matrix.combo.id }}
      # Hoisted to job-level so every step (Sanity, Bucket-A, unsloth_zoo
      # pytest, test_apply_fused_lm_head) inherits it. transformers' bundled
      # *_pb2.py was generated against an older protoc; the C++ protobuf
      # 4+/5+/6 implementation rejects them with "Descriptors cannot be
      # created directly". The pure-Python parser bypasses the check; the
      # speed cost is negligible for these tests.
      PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
      PYTHONPATH: ${{ github.workspace }}/studio
      UNSLOTH_COMPILE_DISABLE: '1'
      # unsloth_zoo/__init__.py:314 raises ImportError unless UNSLOTH_IS_PRESENT
      # is set — normally it is set by unsloth.__init__ when unsloth is imported
      # first. In this job we sometimes import unsloth_zoo.* (e.g.
      # unsloth_zoo.saving_utils, unsloth_zoo.temporary_patches) without going
      # through `import unsloth` first; pin the env var to 1 so unsloth_zoo's
      # bootstrap accepts it. Setting it has no effect on unsloth itself.
      UNSLOTH_IS_PRESENT: '1'
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: '3.12'
          cache: 'pip'

      # Node 22 unblocks tests/studio/test_chat_preset_builtin_invariants.py's
      # `node --experimental-strip-types` subprocess. Cheap to install; keeps
      # the consolidated job self-sufficient even if studio-backend-ci.yml
      # changes its node setup.
      - uses: actions/setup-node@48b55a011bda9f5d6aeb4c2d9c7362e8dae4041e  # v6.4.0
        with:
          node-version: '22'

      - name: Install uv (some unsloth_zoo dev tooling expects it on PATH)
        run: pip install uv

      - name: Resolve matrix specs (handle __from_pyproject__ sentinel)
        # The pyproject cell uses a sentinel; resolve the real `transformers`
        # and `trl` constraints from the project's pyproject.toml at job time.
        # unsloth's pyproject puts the LLM stack pins in
        # [project.optional-dependencies] under the `huggingfacenotorch`
        # extra (top-level [project.dependencies] is just typer/pydantic/etc.),
        # so we walk every optional extra and pick the first matching spec.
        # Other cells pass their spec through unchanged.
        run: |
          set -euxo pipefail
          python <<'PY' >> "$GITHUB_ENV"
          import os, re, tomllib
          spec_t = os.environ["MATRIX_TRANSFORMERS_SPEC"]
          spec_r = os.environ["MATRIX_TRL_SPEC"]

          def _pkg_name(spec: str) -> str:
              m = re.match(r"\s*([A-Za-z0-9_.-]+)", spec)
              return (m.group(1).lower() if m else "")

          if spec_t == "__from_pyproject__" or spec_r == "__from_pyproject__":
              with open("pyproject.toml", "rb") as f:
                  doc = tomllib.load(f)
              proj = doc.get("project", {})
              # Try top-level deps first, then all optional extras.
              all_deps: list[str] = list(proj.get("dependencies", []))
              for _name, dep_list in proj.get("optional-dependencies", {}).items():
                  all_deps.extend(dep_list)

              if spec_t == "__from_pyproject__":
                  spec_t = next((x for x in all_deps if _pkg_name(x) == "transformers"),
                                "transformers")
              if spec_r == "__from_pyproject__":
                  spec_r = next((x for x in all_deps if _pkg_name(x) == "trl"),
                                "trl")
          print(f"RESOLVED_TRANSFORMERS_SPEC={spec_t}")
          print(f"RESOLVED_TRL_SPEC={spec_r}")
          PY
          # Echo to logs so the matrix cell label maps cleanly to a spec.
          grep RESOLVED_ "$GITHUB_ENV" || true

      - name: Install runtime deps (mirrors studio-backend-ci.yml + mlx-ci.yml)
        # The shape matches studio-backend-ci.yml's "Repo tests (CPU)" install
        # so we inherit the same CPU-spoof harness in tests/conftest.py and
        # the same import-chain guarantees, plus the extra deps that the
        # tests/saving + tests/utils Bucket-A files transitively need but
        # which Repo tests (CPU) does not require because it --ignores
        # those directories:
        #   - protobuf + sentencepiece: tests/saving/test_fix_sentencepiece_gguf_robustness.py
        #     does `from transformers.utils import sentencepiece_model_pb2`,
        #     which imports `google.protobuf`. Not pulled by transformers'
        #     base install.
        #   - triton: unsloth/_gpu_init.py:232 does an unconditional
        #     `import triton`. The triton PyPI wheel installs cleanly on
        #     Linux x86_64 even without CUDA (the import succeeds; runtime
        #     GPU work is what would fail, which we never do here).
        # transformers + trl are matrix-parameterized.
        run: |
          set -euxo pipefail
          python -m pip install --upgrade pip
          pip install -r studio/backend/requirements/studio.txt
          pip install \
            python-multipart aiofiles sqlalchemy cryptography \
            pyyaml jinja2 mammoth unpdf requests typer \
            'numpy<3' pytest==9.0.3 pytest-asyncio httpx \
            protobuf sentencepiece triton \
            psutil packaging tqdm safetensors datasets \
            'peft>=0.18,<0.20' 'accelerate>=0.34,<2' \
            ipython
          # torchvision: unsloth_zoo.vision_utils imports it at module scope.
          pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
            'torch>=2.4,<2.11' 'torchvision<0.26'
          # transformers + trl from the matrix combo.
          pip install "$RESOLVED_TRANSFORMERS_SPEC"
          pip install "$RESOLVED_TRL_SPEC"
          # bitsandbytes: hard import in unsloth/models/_utils.py. Recent
          # versions ship a CPU build that imports cleanly on Linux.
          pip install 'bitsandbytes>=0.45'
          # unsloth itself, editable, no-deps so pip does not fight the
          # explicit torch CPU-index install above.
          pip install -e . --no-deps
          echo "::group::Installed transformers + trl + torch + unsloth versions"
          pip show transformers
          pip show trl
          pip show torch
          pip show unsloth
          echo "::endgroup::"

      - name: Clone unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }}
        # We need the repository tree (the wheel does not ship tests/), so
        # clone shallow then editable-install so unsloth_zoo.* imports
        # resolve to the cloned tree. We use `pip show` for the location
        # check rather than `import unsloth_zoo` because the latter calls
        # device_type.get_device_type() at module load and raises on a
        # GPU-less runner; pytest steps below route through the existing
        # tests/conftest.py spoof which handles that.
        run: |
          set -euxo pipefail
          # github.com occasionally 500s on the git fetch; retry so a
          # single upstream blip does not fail CI.
          for attempt in 1 2 3; do
            rm -rf "$RUNNER_TEMP/unsloth-zoo"
            if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \
                https://github.com/unslothai/unsloth-zoo \
                "$RUNNER_TEMP/unsloth-zoo"; then
              break
            fi
            if [ "$attempt" -eq 3 ]; then
              echo "::error::git clone unsloth-zoo failed after 3 attempts"
              exit 1
            fi
            delay=$((5 * attempt))
            echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..."
            sleep "$delay"
          done
          pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps
          pip show unsloth_zoo

      - name: Sanity — collection only (both repos)
        # Catches import-time breakage before we run the suite. Cheap; bails
        # the job out fast if a transformers/torch resolution went sideways.
        # Inherits PYTHONPATH / UNSLOTH_COMPILE_DISABLE / PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION
        # from the job-level env block.
        run: |
          set -euxo pipefail
          python -m pytest --collect-only -q \
            tests/saving/test_save_shell_injection.py \
            tests/saving/test_patch_saving_none_tokenizer.py \
            tests/saving/test_fix_sentencepiece_gguf_robustness.py \
            tests/saving/test_compressed_export_schemes.py \
            tests/saving/test_export_api_surface.py \
            tests/saving/test_export_dispatch.py \
            tests/saving/test_imatrix_export.py \
            tests/utils/test_attention_masks.py \
            tests/utils/test_trunc_normal_patch.py \
            tests/python/test_fast_language_model_text_only.py
          python -m pytest --collect-only -q "$RUNNER_TEMP/unsloth-zoo/tests/"

      - name: import_fixes drift detectors (18 tests, HARD GATE)
        # One drift detector per fix_* / patch_* function in
        # unsloth/import_fixes.py. The detectors assert the *healthy*
        # upstream shape that the fix expects ABSENT the regression;
        # ANY DRIFT DETECTED -> pytest.fail (NEVER skip) so the
        # matrix cell goes red and the maintainer triages on the
        # next PR, not in a downstream user's crash report.
        #
        # Pathologies covered by the suite (each maps to one fix
        # function with the line range cited in the test docstring):
        #   * protobuf MessageFactory GetPrototype / GetMessageClass
        #   * datasets 4.4.x recursion range
        #   * TRL tuple-vs-bool _*_available caching
        #   * transformers PreTrainedModel.enable_input_require_grads
        #     source pattern flip
        #   * transformers torchcodec / causal_conv1d availability
        #     flags
        #   * transformers + accelerate is_wandb_available
        #   * peft.utils.transformers_weight_conversion importability
        #     + build_peft_weight_mapping signature
        #   * triton 3.6+ CompiledKernel num_ctas / cluster_dims
        #   * torch / torchvision pinned compatibility table
        #   * vllm guided_decoding_params / structured_outputs +
        #     aimv2 ovis config version
        #   * huggingface_hub is_offline_mode / HF_HUB_OFFLINE
        #   * torch.nn.init.trunc_normal_ presence (patch site for
        #     patch_trunc_normal_precision_issue)
        #   * xformers post-num_splits-key fix version
        # HARD GATE: a red cell here is a real upstream regression
        # without a corresponding zoo / unsloth-side workaround.
        run: |
          python -m pytest -v --tb=short tests/test_import_fixes_drift.py

      - name: public-api surface drift detectors (9 tests, HARD GATE)
        # Companion to test_import_fixes_drift.py: that file catches
        # third-party drift; this one catches drift in unsloth's OWN
        # public surface (FastLanguageModel / FastVisionModel /
        # FastModel + their classmethods + is_bf16_supported). A
        # rename here would silently break the unslothai/notebooks tree
        # one PR cycle later -- this gate catches it BEFORE the
        # breakage reaches users.
        run: |
          python -m pytest -v --tb=short tests/test_public_api_surface.py

      - name: callback signature drift detector (HARD GATE)
        # Catches the MLX-style bug from PR #5498: a producer in
        # unsloth_zoo (or unsloth) grows a callback arg, but a consumer
        # callback def still declares the old arity. The producer's
        # try/except swallows the resulting TypeError and the symptom is
        # "callback never fires" -- usually diagnosed downstream as a
        # confusing assertion several seconds later. This static AST
        # check fails fast at PR time. UNSLOTH_ZOO_SRC points at the
        # freshly cloned main so the detector sees platform-specific
        # submodules (e.g. unsloth_zoo/mlx/) that the released wheel
        # may strip.
        env:
          UNSLOTH_ZOO_SRC: ${{ runner.temp }}/unsloth-zoo
        run: |
          python -m pytest -v --tb=short tests/test_callback_signature_drift.py

      - name: generation correctness guards (HARD GATE)
        # Deterministic CPU guards, each validated to fail on its pre-fix code:
        # leftpad = batched left-padded generation (#1066/#3699, fixed by
        # #2216 + #4100; staging proof: unsloth-staging-2 PRs 170/172);
        # rope_scaling_drift = config.rope_scaling dropped by replaced rotary
        # classes (#2405). AST checks run first so import breakage cannot mask them.
        run: |
          python -m pytest -v --tb=short \
            tests/utils/test_prepare_inputs_leftpad.py \
            tests/utils/test_rope_scaling_drift.py

      - name: unsloth Bucket-A — CPU tests not in Repo tests (CPU)
        # CPU tests across 6 files under tests/saving/, tests/utils/, tests/python/
        # that Repo tests (CPU) --ignores. AST/protobuf/regex plus tiny CPU model
        # loads; run cleanly here (transformers/torch installed).
        run: |
          python -m pytest -q --tb=short \
            tests/saving/test_save_shell_injection.py \
            tests/saving/test_patch_saving_none_tokenizer.py \
            tests/saving/test_fix_sentencepiece_gguf_robustness.py \
            tests/saving/test_compressed_export_schemes.py \
            tests/saving/test_export_api_surface.py \
            tests/saving/test_export_dispatch.py \
            tests/saving/test_imatrix_export.py \
            tests/utils/test_attention_masks.py \
            tests/utils/test_trunc_normal_patch.py \
            tests/python/test_fast_language_model_text_only.py \
            tests/test_bad_mappings_redirect.py \
            tests/test_prefetch_snapshot_scope.py \
            tests/test_gemma_2b_mapper_key.py \
            --deselect 'tests/utils/test_attention_masks.py::test_run_attention_flash_varlen_receives_window_and_softcap'
          # The deselected test monkeypatches flash_attn_varlen_func, which is
          # only bound on the module when `flash_attn` is importable. flash_attn
          # requires CUDA + dev toolchain, which the CPU-only ubuntu-latest
          # runner does not have. The other Bucket-A tests pass cleanly.

      - name: unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }} — full pytest (CPU)
        # 106 of 111 test_* in unsloth_zoo are CPU-only. The two CUDA-skip
        # cases below auto-skip on a GPU-less runner; deselect them
        # explicitly so the no-CUDA outcome is "deselected", not "skipped",
        # making intent visible in the report. Env inherited from job block.
        #
        # test_get_peft_model_passes_finetune_last_n_layers_through is
        # deselected because unsloth_zoo/mlx/loader.py at line 2972 calls
        # model.trainable_parameters() on the fake-model fixture, which
        # the test never stubbed; this fails on every platform regardless
        # of CUDA. Tracked upstream as an unsloth_zoo bug; deselecting
        # here unblocks unsloth CI until the loader fixture is fixed.
        working-directory: ${{ runner.temp }}/unsloth-zoo
        run: |
          python -m pytest -q --tb=short tests/ \
            --deselect tests/test_unsloth_zoo_lora_merge.py::test_active_merge_device_returns_string_on_cuda_host \
            --deselect tests/test_unsloth_zoo_lora_merge.py::test_merge_lora_moves_cpu_inputs_to_active_device \
            --deselect tests/test_mlx_finetune_last_n_layers.py::test_get_peft_model_passes_finetune_last_n_layers_through

      - name: unsloth_zoo — test_apply_fused_lm_head (lives in compiler.py)
        # `test_apply_fused_lm_head` lives at unsloth_zoo/compiler.py:1983,
        # not under tests/, so pytest's default discovery does not pick it up.
        # We route it through pytest by writing a one-shot shim test file
        # inside the unsloth checkout's tests/ — pytest then walks UP and
        # picks up tests/conftest.py, whose GPU-spoof harness (lines 84-141)
        # patches torch.cuda.is_available, torch.cuda.memory.mem_get_info,
        # torch.cuda.get_device_capability, and is_bf16_supported. That full
        # spoof is required because unsloth_zoo/temporary_patches/gpt_oss.py
        # at module load reads torch.cuda.memory.mem_get_info(0), which
        # bare `is_available = True` doesn't cover. Env inherited.
        run: |
          set -euxo pipefail
          cat > tests/_zoo_apply_fused_lm_head_shim.py <<'PY'
          # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
          # Wraps unsloth_zoo.compiler.test_apply_fused_lm_head so that
          # tests/conftest.py's GPU-spoof harness applies before the import.
          # _zoo_aggressive_cuda_spoof extends conftest's harness with deeper
          # patches (see tests/_zoo_aggressive_cuda_spoof.py).
          import sys, pathlib
          sys.path.insert(0, str(pathlib.Path(__file__).parent))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()
          from unsloth_zoo.compiler import test_apply_fused_lm_head as _zoo_test
          def test_zoo_apply_fused_lm_head_runs():
              _zoo_test()
          PY
          python -m pytest -q --tb=short tests/_zoo_apply_fused_lm_head_shim.py
          rm -f tests/_zoo_apply_fused_lm_head_shim.py

      - name: Static checks — unsloth/trainer.py + unsloth/models/rl.py against latest pip TRL
        # AST-only sanity: confirm both files parse and that every TRL symbol
        # they reference still exists in the installed `trl`. Catches API
        # drift (renamed / removed TRL classes) without running training.
        # Pre-fetches latest pip transformers in case TRL pinned an older one.
        run: |
          set -euxo pipefail
          # Use the matrix-resolved transformers + trl versions already
          # installed by the runtime-deps step (don't upgrade here; that
          # would defeat the matrix's purpose of testing against the
          # specific (transformers, trl) combination the cell selected).
          python <<'PY'
          import ast, importlib, pathlib, sys
          paths = [pathlib.Path("unsloth/trainer.py"),
                   pathlib.Path("unsloth/models/rl.py")]
          for p in paths:
              src = p.read_text()
              tree = ast.parse(src, filename=str(p))
              # Collect every `from trl... import X` and `from trl... import (X, Y)`
              missing = []
              for node in ast.walk(tree):
                  if isinstance(node, ast.ImportFrom) and node.module and node.module.startswith("trl"):
                      mod = importlib.import_module(node.module)
                      for alias in node.names:
                          if alias.name == "*":
                              continue
                          if not hasattr(mod, alias.name):
                              missing.append(f"{node.module}.{alias.name}")
              print(f"{p}: TRL symbols referenced and resolved -> {'OK' if not missing else 'MISSING ' + ', '.join(missing)}")
              if missing:
                  sys.exit(1)
          PY

      - name: Static checks — unsloth_zoo/tiled_mlp.py against latest pip transformers
        # AST parse + transformers symbol-resolution. The user flagged tiled
        # MLP patching as the path that breaks first when transformers ships
        # an MLP class rename; this step is the canary against whatever
        # transformers version the matrix cell selected.
        working-directory: ${{ runner.temp }}/unsloth-zoo
        run: |
          set -euxo pipefail
          python <<'PY'
          import ast, importlib, pathlib, sys
          p = pathlib.Path("unsloth_zoo/tiled_mlp.py")
          src = p.read_text()
          tree = ast.parse(src, filename=str(p))
          missing = []
          for node in ast.walk(tree):
              if isinstance(node, ast.ImportFrom) and node.module and node.module.startswith("transformers"):
                  try:
                      mod = importlib.import_module(node.module)
                  except Exception as e:
                      missing.append(f"{node.module} (import failed: {type(e).__name__})")
                      continue
                  for alias in node.names:
                      if alias.name == "*":
                          continue
                      if not hasattr(mod, alias.name):
                          missing.append(f"{node.module}.{alias.name}")
          print(f"{p}: transformers symbols referenced -> {'OK' if not missing else 'MISSING ' + ', '.join(missing)}")
          if missing:
              sys.exit(1)
          PY

      - name: Static checks — unsloth_zoo/hf_utils.py syntax + import-graph
        working-directory: ${{ runner.temp }}/unsloth-zoo
        run: |
          set -euxo pipefail
          python <<'PY'
          import ast, pathlib
          p = pathlib.Path("unsloth_zoo/hf_utils.py")
          tree = ast.parse(p.read_text(), filename=str(p))
          # Surface every public function + class so the PR check log shows
          # what's covered, not just OK/FAIL.
          public = []
          for node in tree.body:
              if isinstance(node, (ast.FunctionDef, ast.AsyncFunctionDef, ast.ClassDef)) and not node.name.startswith("_"):
                  public.append(f"{type(node).__name__.replace('Def','').lower()}:{node.name}")
          print(f"hf_utils.py public surface ({len(public)}): " + ", ".join(public))
          PY

      - name: Runtime checks — invoke every zero-arg patch_* across both repos (via pytest shim)
        # Routed through pytest so tests/conftest.py's GPU-spoof harness
        # applies before any unsloth_zoo.temporary_patches.* import.
        # Locally validated 50/51 zero-arg patches succeed; the lone failure
        # surfaces a real bug (unsloth.models._utils.patch_fast_lora raises
        # NameError: name 'fast_lora_forward' is not defined). The shim
        # reports the full ledger but only fails when one of the two
        # `required` helpers is absent.
        run: |
          set -euxo pipefail
          cat > tests/_runtime_patch_check_shim.py <<'PY'
          # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
          # Wraps the runtime patch_* validation into a pytest test so the
          # tests/conftest.py GPU-spoof harness applies. continue-on-error
          # at the workflow level catches per-patch failures; this shim only
          # asserts that the two `required` helpers are reachable.
          import sys, pathlib
          sys.path.insert(0, str(pathlib.Path(__file__).parent))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()
          import importlib, inspect

          MODULES = [
              "unsloth.models._utils", "unsloth.models.rl", "unsloth.import_fixes",
              "unsloth.kernels.cross_entropy_loss", "unsloth.kernels.rms_layernorm",
              "unsloth.tokenizer_utils", "unsloth.save",
              "unsloth_zoo.patching_utils", "unsloth_zoo.gradient_checkpointing",
              "unsloth_zoo.loss_utils", "unsloth_zoo.tokenizer_utils",
              "unsloth_zoo.tiled_mlp", "unsloth_zoo.dataset_utils",
              "unsloth_zoo.patch_torch_functions",
              "unsloth_zoo.temporary_patches.gemma",
              "unsloth_zoo.temporary_patches.ministral",
              "unsloth_zoo.temporary_patches.pixtral",
              "unsloth_zoo.temporary_patches.deepseek_v3_moe",
              "unsloth_zoo.temporary_patches.qwen3_5_moe",
              "unsloth_zoo.temporary_patches.mxfp4",
              "unsloth_zoo.temporary_patches.bitsandbytes",
              "unsloth_zoo.temporary_patches.flex_attention_bwd",
          ]
          REQUIRED = {
              "patch_unsloth_smart_gradient_checkpointing",
              "patch_gradient_accumulation_fix",
          }
          # Patches whose signature looks zero-arg (`()` or all-defaulted)
          # but which actually require either runtime args or real CUDA.
          # Calling these in isolation is meaningless, so skip the
          # invocation. Symbol presence (REQUIRED above) is still verified.
          #   patch_linear_scaling / patch_llama_rope_scaling: defaults are
          #     None placeholders; the bodies start with
          #     `assert <param> is not None`.
          #   patch_unsloth_smart_gradient_checkpointing: legitimately
          #     allocates CUDA tensors via aten::empty.memory_format inside
          #     initialize_unsloth_gradient_checkpointing(); the
          #     torch.cuda.* spoof can't intercept that at the dispatcher
          #     level.
          NEEDS_PRECONDITION = {
              "patch_linear_scaling",
              "patch_llama_rope_scaling",
              "patch_unsloth_smart_gradient_checkpointing",
          }

          def test_zero_arg_patch_invocations():
              ok, fail, args, skipped, miss_imports = 0, [], [], [], {}
              seen_required = set()
              for mod_name in MODULES:
                  try:
                      mod = importlib.import_module(mod_name)
                  except Exception as e:
                      miss_imports[mod_name] = f"{type(e).__name__}: {e}"
                      continue
                  for name in sorted(dir(mod)):
                      if not name.startswith("patch_"): continue
                      fn = getattr(mod, name, None)
                      if not callable(fn): continue
                      if name in REQUIRED: seen_required.add(name)
                      try:
                          sig = inspect.signature(fn)
                          need = [p.name for p in sig.parameters.values()
                                  if p.default is inspect.Parameter.empty
                                  and p.kind in (inspect.Parameter.POSITIONAL_OR_KEYWORD,
                                                 inspect.Parameter.POSITIONAL_ONLY)]
                      except (TypeError, ValueError):
                          need = []
                      if need:
                          args.append((mod_name, name, need)); continue
                      if name in NEEDS_PRECONDITION:
                          skipped.append(f"{mod_name}.{name}")
                          print(f"  SKIP {mod_name}.{name} (needs precondition / CUDA)")
                          continue
                      try:
                          fn()
                          ok += 1
                          print(f"  OK   {mod_name}.{name}")
                      except Exception as e:
                          fail.append((mod_name, name, type(e).__name__, str(e)[:200]))
                          print(f"  FAIL {mod_name}.{name} -> {type(e).__name__}: {str(e)[:200]}")
              print(f"\nzero-arg patch_*: ok={ok} fail={len(fail)} skipped={len(skipped)}")
              print(f"arg-required patch_* (skipped, listed for review): {len(args)}")
              for m, n, r in args:
                  print(f"    needs={r}: {m}.{n}")
              if skipped:
                  print(f"explicitly skipped (needs precondition / CUDA): {skipped}")
              if miss_imports:
                  print("\nmodules failed to import (skipped):")
                  for k, v in miss_imports.items():
                      print(f"    {k}: {v}")
              print(f"required patch_* helpers seen: {sorted(seen_required)}")
              missing = REQUIRED - seen_required
              assert not missing, f"required patch_* helpers MISSING: {sorted(missing)}"
              # Strict: any zero-arg patch that raises is a real
              # regression now that #5319 has landed (the three previously
              # known-broken patches are fixed; legitimate
              # CPU-precondition skips are recorded in NEEDS_PRECONDITION
              # above, not in `fail`). Print all failures and re-raise
              # them as one assertion message.
              if fail:
                  raise AssertionError(
                      f"zero-arg patch_* invocation failures (ok={ok}, "
                      f"fail={len(fail)}, skipped={len(skipped)}):\n  "
                      + "\n  ".join(
                          f"{m}.{n} -> {ec}: {msg}" for m, n, ec, msg in fail
                      )
                  )
          PY
          python -m pytest -q --tb=short tests/_runtime_patch_check_shim.py -s
          rm -f tests/_runtime_patch_check_shim.py

      - name: Runtime checks — patch_tiled_mlp on a synthetic MLP module (via pytest shim)
        # Same shim pattern: pytest picks up tests/conftest.py before importing
        # unsloth_zoo.tiled_mlp, so the GPU-spoof harness covers
        # unsloth_zoo.temporary_patches.gpt_oss's mem_get_info call.
        run: |
          set -euxo pipefail
          cat > tests/_tiled_mlp_check_shim.py <<'PY'
          # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
          import sys, pathlib
          sys.path.insert(0, str(pathlib.Path(__file__).parent))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()
          import torch
          import torch.nn as nn
          from unsloth_zoo.tiled_mlp import patch_tiled_mlp, patch_mlp

          class _MLP(nn.Module):
              def __init__(self, hidden=64, intermediate=128):
                  super().__init__()
                  self.gate_proj = nn.Linear(hidden, intermediate, bias=False)
                  self.up_proj   = nn.Linear(hidden, intermediate, bias=False)
                  self.down_proj = nn.Linear(intermediate, hidden, bias=False)
                  self.act_fn = nn.SiLU()
              def forward(self, x):
                  return self.down_proj(self.act_fn(self.gate_proj(x)) * self.up_proj(x))

          class _FakeModel(nn.Module):
              def __init__(self):
                  super().__init__()
                  self.layers = nn.ModuleList([nn.ModuleDict({"mlp": _MLP()}) for _ in range(2)])
              def forward(self, x):
                  for layer in self.layers:
                      x = x + layer["mlp"](x)
                  return x

          def test_patch_tiled_mlp_numerical_equivalence():
              # `patch_mlp(target_arctic=True)` sets `chunk_size = max(1, H)`
              # and shards the SEQUENCE dim with `n_shards = max(1, S //
              # chunk_size)`. Pick S > H so the tiled path actually runs
              # multi-shard (n_shards = 192 // 64 = 3, plus a remainder
              # shard) rather than degenerating to n_shards = 1 which is
              # bit-exact and only confirms patching installed something.
              # If the tiled implementation is correct, multi-shard output
              # must still match the un-tiled reference within FP32 noise.
              torch.manual_seed(0)
              m = _FakeModel().eval()
              hidden = 64
              # 192 = 3 * hidden, so divmod(192, 64) = (3, 0) -> 3 shards,
              # no remainder; gives a clean multi-shard verification.
              x = torch.randn(2, 192, hidden)
              with torch.no_grad():
                  y_before = m(x).clone()
              patch_mlp(m.layers[0]["mlp"])
              patch_tiled_mlp(m)
              # Sanity-check we are actually exercising the multi-shard
              # path: poke chunk_size by re-deriving it the same way
              # `tiled_forward_arctic_size` does.
              S = x.shape[1]
              chunk = max(1, hidden)
              n_shards_expected = max(1, S // chunk)
              assert n_shards_expected > 1, (
                  "tiled MLP shim is not exercising multi-shard: "
                  f"S={S}, chunk={chunk}, n_shards={n_shards_expected}"
              )
              with torch.no_grad():
                  y_after = m(x).clone()
              err = (y_before - y_after).abs().max().item()
              print(
                  f"patch_tiled_mlp multi-shard (n_shards={n_shards_expected}) "
                  f"output diff = {err:.3e}"
              )
              assert err < 1e-3, f"tiled MLP output drifted: {err}"
          PY
          python -m pytest -q --tb=short tests/_tiled_mlp_check_shim.py -s
          rm -f tests/_tiled_mlp_check_shim.py

      - name: Compiler cache hygiene + source-rewriter invariants (synthetic inputs)
        # Lightweight pipeline coverage for unsloth_zoo.compiler. Pure regex
        # / tokenize / ast paths driven by tiny synthetic source strings:
        #   - higher_precision_softmax (basic + idempotent)
        #   - fix_rotary_embedding_dtype (no-op + active under
        #     UNSLOTH_FORCE_CUSTOM_DTYPE)
        #   - fix_attention_dtype_consistency (insert + idempotent)
        #   - convert_attention_masks_to_bool (rewrite + no-op)
        #   - create_new_function happy-path (versioning block, license
        #     header, AST parse, importlib re-import)
        #   - create_new_function **kwargs collision (exercises
        #     _rewrite_kwargs_param + _insert_kwargs_alias)
        #   - UNSLOTH_COMPILE_OVERWRITE=0 forced-recompile on transformers
        #     version mismatch (compiler.py:947-963)
        #   - matching short-circuit when versions are equal
        # No real transformers modeling module is loaded; complements the
        # heavier real-class round-trip step below. Wall-time ~10-25s.
        run: |
          set -euxo pipefail
          cat > tests/_compiler_cache_invariants_shim.py <<'PY'
          # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
          # Cache-hygiene + source-rewriter invariants for unsloth_zoo.compiler.
          import sys, pathlib, os, ast, importlib, importlib.util, time
          sys.path.insert(0, str(pathlib.Path(__file__).parent))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()
          import pytest
          import torch  # noqa: F401  (compiler.py imports torch at module load)


          def _isolate_cache(tmp_path, monkeypatch):
              """Point UNSLOTH_COMPILE_LOCATION at tmp_path and reset module
              globals. The compiler.py global is captured at module load
              (line 75/179), so we delete + reimport per test."""
              monkeypatch.setenv("UNSLOTH_COMPILE_LOCATION", str(tmp_path))
              if "unsloth_zoo.compiler" in sys.modules:
                  del sys.modules["unsloth_zoo.compiler"]
              import unsloth_zoo.compiler as compiler
              compiler.UNSLOTH_COMPILE_LOCATION = str(tmp_path)
              compiler.UNSLOTH_COMPILE_USE_TEMP = False
              return compiler


          def test_higher_precision_softmax_basic_and_idempotent(tmp_path, monkeypatch):
              c = _isolate_cache(tmp_path, monkeypatch)
              src = (
                  "y = nn.functional.softmax(x, dim=-1)\n"
                  "z = F.softmax(a, dim=1, dtype=torch.bfloat16)\n"
              )
              out = c.higher_precision_softmax(src)
              assert "dtype = torch.float32).to(x.dtype)" in out
              assert "dtype = torch.float32).to(a.dtype)" in out
              # Idempotency landed in unslothai/unsloth-zoo#631
              # (negative-lookahead on `.to(<var>.dtype)` so a second
              # pass does not append another cast).
              assert c.higher_precision_softmax(out) == out


          def test_fix_rotary_dtype_no_op_without_env(tmp_path, monkeypatch):
              c = _isolate_cache(tmp_path, monkeypatch)
              monkeypatch.delenv("UNSLOTH_FORCE_CUSTOM_DTYPE", raising=False)
              src = "out = cos.to(dtype=x.dtype) + sin.to(dtype=x.dtype)\n"
              assert c.fix_rotary_embedding_dtype(src) == src


          def test_fix_rotary_dtype_active(tmp_path, monkeypatch):
              c = _isolate_cache(tmp_path, monkeypatch)
              monkeypatch.setenv(
                  "UNSLOTH_FORCE_CUSTOM_DTYPE",
                  "float16;torch.float32;torch.bfloat16;torch.float16;pass",
              )
              monkeypatch.setenv("UNSLOTH_FORCE_FLOAT32", "1")
              src = "out = cos.to(dtype=x.dtype) + sin.to(dtype=x.dtype)\n"
              out = c.fix_rotary_embedding_dtype(src)
              # Active form rewrites cos.to / sin.to. Either the conditional
              # form or the cast form is acceptable -- different transformers
              # versions surface slightly different outputs from the rewriter.
              assert "cos.to(dtype=x.dtype)" not in out
              assert "sin.to(dtype=x.dtype)" not in out


          def test_fix_attention_dtype_consistency_insert_then_idempotent(tmp_path, monkeypatch):
              c = _isolate_cache(tmp_path, monkeypatch)
              src = (
                  "    query_states, key_states = apply_rotary_pos_emb("
                  "query_states, key_states, cos, sin)\n"
                  "    attn = q @ k.T\n"
              )
              out = c.fix_attention_dtype_consistency(src)
              assert out.count("value_states = value_states.to(query_states.dtype)") == 1
              assert c.fix_attention_dtype_consistency(out) == out


          def test_convert_attention_masks_to_bool_rewrites(tmp_path, monkeypatch):
              c = _isolate_cache(tmp_path, monkeypatch)
              src = (
                  "def make_mask(x):\n"
                  "    out = torch.finfo(x.dtype).min * x\n"
                  "    return out\n"
              )
              out = c.convert_attention_masks_to_bool("make_mask", src)
              # Loose match: rewriter inserts a `!=torch.finfo(...).min` check
              # somewhere on the return path. Tightening to an exact
              # last-line match is brittle across transformers versions.
              assert "!=torch.finfo" in out


          def test_convert_attention_masks_to_bool_no_op(tmp_path, monkeypatch):
              c = _isolate_cache(tmp_path, monkeypatch)
              src = "def make_mask(x):\n    return x\n"
              assert c.convert_attention_masks_to_bool("make_mask", src) == src


          def _versioning_lines(file_text):
              """Extract the four version strings from the versioning block."""
              assert file_text.startswith('"""\n'), "missing opening triple-quote"
              head = file_text.split("__UNSLOTH_VERSIONING__", 1)[0]
              lines = [ln for ln in head.splitlines() if ln and ln != '"""']
              return lines


          def test_create_new_function_happy_path(tmp_path, monkeypatch):
              c = _isolate_cache(tmp_path, monkeypatch)
              src = "def f(x):\n    return nn.functional.softmax(x, dim=-1)\n"
              c.create_new_function(
                  name="f_happy", new_source=src, model_location="builtins",
                  functions=[], overwrite=True,
              )
              cached = tmp_path / "f_happy.py"
              assert cached.exists()
              text = cached.read_text(encoding="utf-8")
              versions = _versioning_lines(text)
              assert len(versions) == 4, versions
              assert text.count(c._full_license_header) == 1
              ast.parse(text)
              spec = importlib.util.spec_from_file_location("f_happy_reimport", cached)
              m2 = importlib.util.module_from_spec(spec)
              spec.loader.exec_module(m2)
              assert callable(m2.f)
              import inspect as _inspect
              # higher_precision_softmax should have promoted to float32.
              assert "dtype = torch.float32" in _inspect.getsource(m2.f)


          def test_create_new_function_overwrite_zero_recompiles_on_version_mismatch(
              tmp_path, monkeypatch,
          ):
              c = _isolate_cache(tmp_path, monkeypatch)
              name = "vmismatch"
              cached = tmp_path / f"{name}.py"
              stub = (
                  '"""\n0.0.0\n0.0.0\n0.0.0-stub\n0.0.0\n__UNSLOTH_VERSIONING__\n"""\n'
                  + c._full_license_header
                  + "def vmismatch(x):\n    return x\n"
              )
              cached.write_text(stub, encoding="utf-8")
              monkeypatch.setenv("UNSLOTH_COMPILE_OVERWRITE", "0")
              src = "def vmismatch(x):\n    return x + 1\n"
              c.create_new_function(
                  name=name, new_source=src, model_location="builtins",
                  functions=[], overwrite=False,
              )
              text = cached.read_text(encoding="utf-8")
              assert "0.0.0-stub" not in text, (
                  "OVERWRITE=0 + transformers-version-mismatch did NOT recompile"
              )
              versions = _versioning_lines(text)
              import importlib.metadata as _md
              assert versions[2] == _md.version("transformers")


          def test_create_new_function_overwrite_zero_short_circuits_when_versions_match(
              tmp_path, monkeypatch,
          ):
              c = _isolate_cache(tmp_path, monkeypatch)
              name = "vmatch"
              src = "def vmatch(x):\n    return x\n"
              c.create_new_function(
                  name=name, new_source=src, model_location="builtins",
                  functions=[], overwrite=True,
              )
              cached = tmp_path / f"{name}.py"
              mtime_before = cached.stat().st_mtime_ns
              time.sleep(0.05)
              monkeypatch.setenv("UNSLOTH_COMPILE_OVERWRITE", "0")
              c.create_new_function(
                  name=name, new_source=src, model_location="builtins",
                  functions=[], overwrite=False,
              )
              assert cached.stat().st_mtime_ns == mtime_before, (
                  "OVERWRITE=0 + matching versions should NOT rewrite the file"
              )
          PY
          python -m pytest -q --tb=short tests/_compiler_cache_invariants_shim.py
          rm -f tests/_compiler_cache_invariants_shim.py

      - name: Compiler full-model-sweep (every transformers.models.*) + SFT trainer round-trip
        # Calls `unsloth_compile_transformers(model_type=...)` against EVERY
        # `transformers.models.<x>` package the matrix's transformers ships
        # (pkgutil.iter_modules walk -- 383 packages on 4.57.6, similar on
        # latest), then ast.parse / importlib-load / introspect the
        # generated unsloth_compiled_cache/*.py file per model. Catches
        # regex / source-rewriter drift across the matrix's (transformers,
        # trl) combination -- the dominant failure mode of
        # `unsloth_compile_transformers` after a transformers point release.
        #
        # 21 model_types currently break the compiler (verified locally on
        # transformers 4.57.6). They are listed in KNOWN_BROKEN below with
        # their failure mode so the sweep stays green and any NEW breakage
        # surfaces as red. Each entry is tracked for an individual fix
        # PR on unsloth-zoo. The list is split by failure category so
        # follow-up PRs can target one bug at a time.
        #
        # Hermetic cache dir per pytest invocation; we override the
        # job-level UNSLOTH_COMPILE_DISABLE=1 inside the shim so
        # compilation actually runs here. Wall-time estimate ~2-3 min
        # warm (mean ~0.3s/model, 383 models = ~110s on the runner).
        run: |
          set -euxo pipefail
          cat > tests/_zoo_compiler_cache_shim.py <<'PY'
          # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
          import os, sys, ast, pathlib, importlib.util, tempfile
          _HERE = pathlib.Path(__file__).parent
          sys.path.insert(0, str(_HERE))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()

          # Hermetic cache dir + force compile path. The compiler's
          # globals (UNSLOTH_COMPILE_LOCATION, UNSLOTH_COMPILE_USE_TEMP)
          # are captured at module load; an earlier conftest `import
          # unsloth` may have already imported unsloth_zoo.compiler with
          # the default "unsloth_compiled_cache" path. Mutate the live
          # module globals after import so this shim is robust to that
          # ordering. Otherwise the compiler silently writes to the
          # default cache and the per-model file assertion fails.
          _CACHE = pathlib.Path(tempfile.mkdtemp(prefix="unsloth_cache_"))
          os.environ["UNSLOTH_COMPILE_LOCATION"] = str(_CACHE)
          os.environ["UNSLOTH_COMPILE_OVERWRITE"] = "1"
          os.environ.pop("UNSLOTH_COMPILE_DISABLE", None)

          import pytest
          import unsloth_zoo.compiler as _zoo_compiler
          _zoo_compiler.UNSLOTH_COMPILE_LOCATION = str(_CACHE)
          _zoo_compiler.UNSLOTH_COMPILE_USE_TEMP = False
          from unsloth_zoo.compiler import unsloth_compile_transformers


          def _verify_file(path: pathlib.Path, must_expose):
              assert path.exists(), f"compiler did not write {path}"
              src = path.read_text(encoding="utf-8")
              ast.parse(src, filename=str(path))
              spec = importlib.util.spec_from_file_location(path.stem, path)
              mod = importlib.util.module_from_spec(spec)
              spec.loader.exec_module(mod)
              for name in must_expose:
                  assert hasattr(mod, name), (
                      f"{path.name} missing expected attr {name!r}; "
                      f"found: {sorted(n for n in dir(mod) if not n.startswith('_'))[:25]}"
                  )


          # ---------- Full transformers.models.* compile sweep ----------
          # Track the model_types that currently break the compiler on
          # transformers >=5,<6. After unsloth-zoo#632 landed, transformers
          # 4.57.6 has zero failures across all model_types; the 27 entries
          # below are the residual failures on the tf 5.x line. New breakage
          # on any OTHER model_type fails the cell. Each entry is a
          # tracking item for a follow-up unsloth-zoo PR.
          KNOWN_BROKEN_COMPILE = {
              # Category A: `string index out of range` in source rewriter.
              "colpali":         "string index out of range",
              "colqwen2":        "string index out of range",
              "colmodernvbert":  "string index out of range",
              "dpr":             "string index out of range",
              "gemma4_assistant":"string index out of range",
              "rag":             "string index out of range",
              "shieldgemma2":    "string index out of range",
              "timm_backbone":   "string index out of range",
              # Category B: rewriter emits invalid Python source.
              "clvp":            "emitted file: unexpected indent",
              "falcon_mamba":    "emitted file: unexpected indent",
              "gpt2":            "emitted file: unexpected indent",
              "imagegpt":        "emitted file: unexpected indent",
              "mamba":           "emitted file: unexpected indent",
              "tapas":           "emitted file: expected ':'",
              "xlstm":           "emitted file: unexpected indent",
              # Category B-2: emit unterminated string literal (latest tf).
              "audioflamingo3":  "emitted file: unterminated string literal",
              "musicflamingo":   "emitted file: unterminated string literal",
              "voxtral":         "emitted file: unterminated string literal",
              "voxtral_realtime":"emitted file: unterminated string literal",
              # Category C: rewriter emits unclosed paren.
              "kosmos2":         "emitted file: '(' was never closed",
              "kosmos2_5":       "emitted file: '(' was never closed",
              # Category D: imports list builder picks up a non-exported name.
              "auto":            "module has no attribute _BaseModelWithGenerate",
              "bit":             "module has no attribute Linear",
              "regnet":          "module has no attribute Linear",
              "resnet":          "module has no attribute Linear",
              # Category E: undefined name in emitted file.
              "perceiver":       "name 'AbstractPreprocessor' is not defined",
              "sam3_lite_text":  "name 'Sam3LiteTextLayerScaledResidual' is not defined",
              # Category F: compile exceeds 60s budget on the runner.
              # First seen on transformers >=5,<6; each represents a slow
              # or recursive source-rewriter path the zoo can address.
              "beit":            "TimeoutError: compile exceeds per-model budget",
              "deepseek_ocr2":    "TimeoutError: compile exceeds per-model budget",
              "sam":             "TimeoutError: compile exceeds per-model budget",
              "sam_hq":          "TimeoutError: compile exceeds per-model budget",
              "deepseek_ocr2":   "TimeoutError: compile exceeds per-model budget",
          }


          def _all_model_types():
              import pkgutil, transformers.models as tm
              return sorted(s.name for s in pkgutil.iter_modules(tm.__path__) if s.ispkg)


          def test_compile_every_transformers_model_type():
              """Run unsloth_compile_transformers across every model_type
              the matrix's transformers ships. Allowed outcomes:
                ok      -> compile emitted a parseable, importable cache file
                skipped -> no `modeling_<x>.py` file (expected for some
                           umbrella packages like `auto`, `deprecated`)
                known   -> in KNOWN_BROKEN_COMPILE; tracked for follow-up.
              Any uncaught failure fails the cell.

              Per-model SIGALRM cap so one infinite-looping model_type
              cannot wedge the whole sweep + nuke the job timeout
              (observed on transformers >=5,<6 -- 30+ min hang before
              this guard landed)."""
              import importlib as _il
              import signal
              ok = 0
              skipped = []
              known = []
              new_failures = []
              models = _all_model_types()
              def _on_timeout(signum, frame):
                  raise TimeoutError("compile exceeded per-model budget")
              prev_handler = signal.signal(signal.SIGALRM, _on_timeout)
              try:
                  for i, model_type in enumerate(models):
                      if i % 25 == 0:
                          print(f"  sweep progress: {i}/{len(models)} -> {model_type}", flush=True)
                      modeling_path = f"transformers.models.{model_type}.modeling_{model_type}"
                      try:
                          _il.import_module(modeling_path)
                      except (ModuleNotFoundError, ImportError):
                          skipped.append((model_type, "no modeling file"))
                          continue
                      signal.alarm(60)
                      try:
                          unsloth_compile_transformers(
                              model_type=model_type, fast_lora_forwards=False,
                          )
                      except Exception as e:
                          signal.alarm(0)
                          msg = f"{type(e).__name__}: {str(e)[:200]}"
                          if model_type in KNOWN_BROKEN_COMPILE:
                              known.append((model_type, msg))
                          else:
                              new_failures.append((model_type, msg))
                          continue
                      signal.alarm(0)
                      if model_type in KNOWN_BROKEN_COMPILE:
                          # Came back green unexpectedly -- that's GOOD news,
                          # the bug was fixed. Surface it so we can drop the
                          # entry from KNOWN_BROKEN_COMPILE.
                          print(
                              f"  UNEXPECTED-OK {model_type}: was in "
                              "KNOWN_BROKEN_COMPILE, now compiles cleanly. "
                              "Drop the entry."
                          )
                      ok += 1
              finally:
                  signal.alarm(0)
                  signal.signal(signal.SIGALRM, prev_handler)
              print(f"\nCompile sweep: ok={ok} skipped={len(skipped)} "
                    f"known-broken={len(known)} new-failures={len(new_failures)}")
              for m, r in known:
                  print(f"  KNOWN  {m}: {r}")
              for m, r in new_failures[:30]:
                  print(f"  NEW    {m}: {r}")
              if len(new_failures) > 30:
                  print(f"  ...and {len(new_failures)-30} more new failures")
              assert not new_failures, (
                  f"unsloth_compile_transformers introduced new failures on "
                  f"{len(new_failures)} model_types not in the known-broken "
                  f"list: {[m for m, _ in new_failures]}"
              )
              # Sanity floor: at least 200 model_types should compile cleanly
              # (we observed 362 ok / 383 total on transformers 4.57.6).
              assert ok >= 200, (
                  f"only {ok} model_types compiled cleanly; expected >=200. "
                  "Possible transformers-version-induced regression."
              )


          @pytest.mark.parametrize("model_type,rms_class", [
              ("llama", "LlamaRMSNorm"),
              ("qwen3", "Qwen3RMSNorm"),
              ("gemma3", "Gemma3RMSNorm"),
          ])
          def test_compile_real_modeling_module(model_type, rms_class):
              """Spot-check on the three production-relevant families that
              the compile_every sweep also covers; this case verifies the
              emitted cache file has the model-specific RMSNorm class
              attribute, not just that the file parses + imports.

              ``unsloth_compile_transformers`` is not idempotent in-
              process: calling it twice on the same modeling module
              after rewriting class attributes corrupts the inspect
              source/line cache and the second emitted file is malformed
              Python. The sweep above already produced a valid cache
              file for every non-KNOWN_BROKEN model_type, so just verify
              that artefact here. Trigger a compile only when running
              this test in isolation (no sweep preceded)."""
              import importlib as _il
              try:
                  modeling = _il.import_module(
                      f"transformers.models.{model_type}.modeling_{model_type}"
                  )
              except ModuleNotFoundError:
                  pytest.skip(
                      f"transformers build lacks model_type={model_type}"
                  )
              combined = _CACHE / f"unsloth_compiled_module_{model_type}.py"
              if not combined.exists():
                  unsloth_compile_transformers(
                      model_type=model_type, fast_lora_forwards=False,
                  )
                  modeling = _il.import_module(
                      f"transformers.models.{model_type}.modeling_{model_type}"
                  )
              assert getattr(modeling, "__UNSLOTH_PATCHED__", False) is True
              _verify_file(combined, must_expose=[rms_class])


          def test_compile_disable_writes_nothing():
              """Negative control: when UNSLOTH_COMPILE_DISABLE=1 the
              compile path must early-return without producing new files."""
              os.environ["UNSLOTH_COMPILE_DISABLE"] = "1"
              try:
                  before = set(_CACHE.iterdir())
                  # Pick a model_type that still resolves on this transformers.
                  for mt in ("llama", "mistral", "qwen2"):
                      try:
                          import importlib as _il
                          _il.import_module(
                              f"transformers.models.{mt}.modeling_{mt}"
                          )
                          break
                      except ModuleNotFoundError:
                          continue
                  else:
                      pytest.skip("no probe model_type available")
                  unsloth_compile_transformers(
                      model_type=mt, fast_lora_forwards=False,
                  )
                  after = set(_CACHE.iterdir())
                  assert after == before, (
                      f"DISABLE=1 still wrote: {[p.name for p in after - before]}"
                  )
              finally:
                  os.environ.pop("UNSLOTH_COMPILE_DISABLE", None)


          def test_compile_sft_trainer_patch():
              """Round-trip TRL's SFTTrainer through the rl.py patch path
              and verify the generated UnslothSFTTrainer.py."""
              pytest.importorskip("trl")
              try:
                  from unsloth.models.rl import _patch_trl_rl_trainers
              except ImportError:
                  pytest.skip("unsloth.models.rl._patch_trl_rl_trainers absent")
              try:
                  _patch_trl_rl_trainers("sft_trainer")
              except Exception as e:
                  # TRL 1.x renames break the patch helper internally; we
                  # accept that here and skip rather than fail the cell.
                  pytest.skip(f"_patch_trl_rl_trainers raised: {type(e).__name__}: {e}")
              sft = _CACHE / "UnslothSFTTrainer.py"
              if not sft.exists():
                  pytest.skip(
                      "_patch_trl_rl_trainers ran but did not emit "
                      "UnslothSFTTrainer.py on this TRL version."
                  )
              _verify_file(sft, must_expose=["UnslothSFTTrainer"])
          PY
          python -m pytest -q --tb=short tests/_zoo_compiler_cache_shim.py
          rm -f tests/_zoo_compiler_cache_shim.py

      - name: TRL trainer + Config auto-discovery + dynamic patch coverage
        # Mirror unsloth/models/rl.py:patch_trl_rl_trainers AND verify the
        # dynamic per-version patch surface:
        #   1. AST-parse every *_trainer / *_config submodule.
        #   2. Apply the same *Trainer / *Config discovery rules
        #      _patch_trl_rl_trainers uses (rl.py:553-620).
        #   3. Orphan check: every <x>_trainer must have a sibling
        #      <x>_config OR an inline *Config.
        #   4. Dynamic count: enumerate every canonical trainer that
        #      imports cleanly, run patch_trl_rl_trainers(), assert
        #      every one ends up Unsloth-prefixed in-place. Floor matches
        #      the cohort sizes from the version sweep:
        #        TRL 0.22-0.23 -> 18 canonical trainers
        #        TRL 0.24-0.28 -> 15 canonical trainers
        #        TRL 0.29-1.x  ->  6 canonical (rest are experimental
        #                          thin-wrappers; covered next)
        #   5. Experimental coverage (TRL 0.29+): walk trl.experimental.*,
        #      find every *Trainer class, verify the umbrella patch
        #      reaches them via the thin-wrapper MRO walk in
        #      _patch_trl_rl_trainers (rl.py:677-702).
        # Per-cell wall-time ~30-60s.
        run: |
          set -euxo pipefail
          cat > tests/_trl_trainer_discovery_shim.py <<'PY'
          # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
          # Walks every *_trainer / *_config module in trl.trainer and
          # validates that unsloth's auto-discovery rules in
          # unsloth/models/rl.py:_patch_trl_rl_trainers (lines 542-620,
          # 1934-1949) still pick out exactly one *Trainer and one
          # *Config per module on the matrix's TRL version.
          import sys, pathlib, importlib, importlib.util, ast, inspect

          sys.path.insert(0, str(pathlib.Path(__file__).parent))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()

          import pytest
          pytest.importorskip("trl")
          import trl  # noqa: F401  (forces lazy-module init)
          import trl.trainer


          def _is_real_submodule(qual_name: str) -> bool:
              """True iff `qual_name` resolves to an importable submodule
              with a file on disk (i.e. has a non-None find_spec().origin).

              TRL re-exports utility FUNCTIONS into `trl.trainer.__init__`
              whose names happen to end with `_config` (e.g.
              `get_peft_config`, `get_quantization_config`). Without this
              filter the `endswith` check below picks them up as if they
              were submodules and the AST stage fails on `no spec`. The
              same trap exists for `_trainer` (none today, but defensive).
              """
              try:
                  spec = importlib.util.find_spec(qual_name)
              except (ImportError, ValueError):
                  return False
              return spec is not None and bool(getattr(spec, "origin", None))


          # Replicate rl.py:1939-1943 verbatim, then filter to actual
          # submodules so re-exported utility functions (e.g.
          # `get_peft_config`) do not pollute the AST sweep.
          def _trainer_files():
              return [
                  x for x in dir(trl.trainer)
                  if x.islower()
                  and x.endswith("_trainer")
                  and x != "base_trainer"
                  and _is_real_submodule(f"trl.trainer.{x}")
              ]


          def _config_files():
              return [
                  x for x in dir(trl.trainer)
                  if x.islower()
                  and x.endswith("_config")
                  and _is_real_submodule(f"trl.trainer.{x}")
              ]


          def _ast_parse_module_via_spec(qual_name: str):
              """AST-parse a module's source on disk WITHOUT importing it.
              `trl.trainer` uses _LazyModule so `find_spec` resolves the
              file path without firing the module-level `__init__`. This
              dodges optional-dep ImportErrors (e.g. grpo_trainer's vllm
              import) and still surfaces real syntax drift in the file."""
              spec = importlib.util.find_spec(qual_name)
              if spec is None or not spec.origin:
                  return None, "no spec"
              path = pathlib.Path(spec.origin)
              if not path.is_file():
                  return None, f"spec.origin not a file: {path}"
              src = path.read_text(encoding="utf-8")
              ast.parse(src, filename=str(path))
              return path, None


          def test_every_trl_trainer_and_config_module_ast_parses():
              """Stage 1: pure file-on-disk AST parse. Catches a TRL
              source-level syntax issue on any matrix cell without
              triggering optional-dep imports."""
              fail = []
              ok = 0
              for name in _trainer_files() + _config_files():
                  qual = f"trl.trainer.{name}"
                  try:
                      path, err = _ast_parse_module_via_spec(qual)
                      if err:
                          fail.append((qual, err))
                      else:
                          ok += 1
                  except SyntaxError as e:
                      fail.append((qual, f"SyntaxError: {e}"))
                  except Exception as e:
                      fail.append((qual, f"{type(e).__name__}: {e}"))
              print(f"AST-parsed {ok} TRL trainer+config modules; failed={len(fail)}")
              for q, e in fail:
                  print(f"  AST FAIL {q}: {e}")
              assert not fail, f"AST parse failed for {len(fail)} TRL modules"


          def _apply_unsloth_discovery_rules(mod, trainer_file):
              """Replicate the four endswith filters in
              rl.py:553-569 verbatim."""
              prefix = trainer_file.split("_")[0]
              names = [
                  x for x in dir(mod)
                  if x.endswith("Trainer") and x != "Trainer"
                  and not x.startswith("_") and prefix in x.lower()
              ]
              configs = [
                  x for x in dir(mod)
                  if x.endswith("Config") and x != "Config"
                  and not x.startswith("_") and prefix in x.lower()
              ]
              return names, configs


          def _resolve_config_via_fallbacks(trainer_file, name_list, mod):
              """Replicate rl.py:575-615: try the sibling *_config.py
              module, then the MRO walk fallback. Returns the resolved
              config-name list (length 0 or 1)."""
              # Fallback 1: <prefix>_config.py module sibling.
              cfg_module_name = trainer_file.replace("_trainer", "_config")
              try:
                  cfg_mod = getattr(trl.trainer, cfg_module_name)
              except Exception:
                  cfg_mod = None
              if cfg_mod is not None:
                  prefix = trainer_file.split("_")[0]
                  hits = [
                      x for x in dir(cfg_mod)
                      if x.endswith("Config") and x != "Config"
                      and not x.startswith("_") and prefix in x.lower()
                  ]
                  if len(hits) == 1:
                      return hits
              # Fallback 2: MRO walk into experimental parent module.
              if len(name_list) != 1:
                  return []
              try:
                  trainer_cls = getattr(mod, name_list[0])
              except Exception:
                  return []
              prefix = trainer_file.split("_")[0]
              for parent in trainer_cls.__mro__[1:]:
                  if parent is object:
                      continue
                  parent_mod = inspect.getmodule(parent)
                  if parent_mod is None:
                      continue
                  if parent_mod.__name__ == f"trl.trainer.{trainer_file}":
                      continue
                  hits = [
                      x for x in dir(parent_mod)
                      if x.endswith("Config") and x != "Config"
                      and not x.startswith("_") and prefix in x.lower()
                  ]
                  if len(hits) == 1:
                      return hits
              return []


          def test_unsloth_auto_discovery_finds_trainer_and_config_per_module():
              """Stage 2: drive the same unsloth rules over every trainer
              file. import-failures (optional deps) are recorded as
              `import-skipped`, mirroring rl.py:1944-1948 try/except."""
              ok = 0
              import_skipped = []
              discovery_skipped = []
              fail = []
              for trainer_file in _trainer_files():
                  qual = f"trl.trainer.{trainer_file}"
                  try:
                      mod = getattr(trl.trainer, trainer_file)
                  except Exception as e:
                      import_skipped.append((qual, f"{type(e).__name__}: {e}"))
                      continue
                  trainers, configs = _apply_unsloth_discovery_rules(
                      mod, trainer_file,
                  )
                  if len(trainers) != 1:
                      discovery_skipped.append(
                          (qual, f"trainers={trainers}")
                      )
                      continue
                  if len(configs) != 1:
                      configs = _resolve_config_via_fallbacks(
                          trainer_file, trainers, mod,
                      )
                  if len(configs) != 1:
                      fail.append(
                          (qual,
                           f"trainer={trainers[0]} but config not found "
                           "(checked module, *_config sibling, and MRO)")
                      )
                      continue
                  ok += 1
                  print(f"  OK {qual}: trainer={trainers[0]}, config={configs[0]}")
              print(
                  f"\nDiscovery: ok={ok} import_skipped={len(import_skipped)} "
                  f"discovery_skipped={len(discovery_skipped)} fail={len(fail)}"
              )
              for q, r in import_skipped:
                  print(f"  IMPORT-SKIP {q}: {r}")
              for q, r in discovery_skipped:
                  print(f"  DISC-SKIP   {q}: {r}")
              for q, r in fail:
                  print(f"  FAIL        {q}: {r}")
              # Hard contract: every TRAINER that imports cleanly AND has
              # exactly one *Trainer must also resolve exactly one *Config
              # via one of the three rules. import-skipped + discovery-
              # skipped (no/multiple *Trainer) are tolerated.
              assert not fail, (
                  f"unsloth discovery rules failed for {len(fail)} trainers"
              )
              # Sanity: at least 3 trainers should fully discover on any
              # matrix cell (sft + reward + dpo are the historical core).
              assert ok >= 3, (
                  f"only {ok} trainers fully discovered; expected >=3 "
                  "(sft/reward/dpo). Possible TRL surface regression."
              )


          def test_orphan_trainer_modules_do_not_exist():
              """Stage 3: every <x>_trainer module should have a sibling
              <x>_config (TRL 0.26+ convention) OR an inline *Config. An
              ORPHAN <x>_trainer with neither is a TRL refactor we want
              to know about: it would silently break unsloth's
              auto-discovery without raising."""
              orphans = []
              for trainer_file in _trainer_files():
                  cfg_module_name = trainer_file.replace("_trainer", "_config")
                  has_sibling_cfg = (
                      importlib.util.find_spec(
                          f"trl.trainer.{cfg_module_name}"
                      ) is not None
                  )
                  if has_sibling_cfg:
                      continue
                  # No sibling -> require an inline *Config in the
                  # trainer module itself (resolved via discovery rules).
                  try:
                      mod = getattr(trl.trainer, trainer_file)
                  except Exception:
                      # Optional-dep failure -> skip; the AST-parse stage
                      # already covered the file.
                      continue
                  _, configs = _apply_unsloth_discovery_rules(
                      mod, trainer_file,
                  )
                  if not configs:
                      orphans.append(trainer_file)
              assert not orphans, (
                  "Orphan TRL trainer modules with neither sibling "
                  f"<x>_config.py nor an inline *Config: {orphans}. "
                  "unsloth auto-discovery would silently skip these."
              )


          # ---- Dynamic patch coverage: count + verify Unsloth-prefixed ----

          def _enumerate_canonical_trainer_classes():
              """Walk trl.trainer/*_trainer.py on disk (the source of
              truth for what `dir(trl.trainer)` should expose) and return
              [(trainer_file, TrainerClass), ...] for every entry that
              imports + has exactly-one resolvable *Trainer per the
              unsloth rules. Skips optional-dep ImportErrors."""
              out = []
              for trainer_file in _trainer_files():
                  try:
                      mod = getattr(trl.trainer, trainer_file)
                  except Exception:
                      continue
                  trainers, _ = _apply_unsloth_discovery_rules(mod, trainer_file)
                  if len(trainers) != 1:
                      continue
                  try:
                      cls = getattr(mod, trainers[0])
                  except Exception:
                      continue
                  out.append((trainer_file, cls))
              return out


          def _enumerate_experimental_trainer_packages():
              """TRL 0.29+ moved many trainers (bco, cpo, gkd, nash_md,
              online_dpo, orpo, ppo, prm, xpo, ...) to `trl.experimental.<pkg>`,
              re-exposing them via thin-wrapper deprecation shims in
              `trl.trainer.<x>_trainer`. List every `trl.experimental.<pkg>`
              that defines at least one *Trainer class, parsed by AST so we
              do NOT trigger the optional-dep imports on the package init."""
              spec = importlib.util.find_spec("trl.experimental")
              if spec is None or not spec.submodule_search_locations:
                  return []
              import re as _re
              hits = []
              for root in spec.submodule_search_locations:
                  rp = pathlib.Path(root)
                  for sub in sorted(rp.iterdir()):
                      if not sub.is_dir() or sub.name.startswith("_"):
                          continue
                      classes = []
                      for py in sub.rglob("*.py"):
                          try:
                              src = py.read_text(encoding="utf-8")
                          except Exception:
                              continue
                          for m in _re.finditer(
                              r"^class\s+([A-Za-z0-9_]+Trainer)\b", src, _re.M,
                          ):
                              classes.append(m.group(1))
                      if classes:
                          hits.append((sub.name, sorted(set(classes))))
              return hits


          def _is_unsloth_patched(cls) -> bool:
              return getattr(cls, "__name__", "").startswith("Unsloth")


          def test_unsloth_patches_every_canonical_trainer_in_this_trl_version():
              """Verify the count + identity of canonically-patched trainers
              matches the trainer surface this TRL version actually ships.

              For TRL 0.22.x-0.23.x: ~18 canonical trainers expected.
              For TRL 0.24.x-0.28.x: ~15 canonical trainers expected.
              For TRL 0.29.x-1.x:    6 canonical (rest are experimental
              thin-wrappers; covered by the next test)."""
              from unsloth.models.rl import patch_trl_rl_trainers
              before = _enumerate_canonical_trainer_classes()
              before_count = len(before)
              before_unpatched = [
                  (tf, cls.__name__) for tf, cls in before
                  if not _is_unsloth_patched(cls)
              ]
              # Apply unsloth's umbrella patch.
              patch_trl_rl_trainers()
              # Re-enumerate (some classes may have been replaced in-module).
              after = _enumerate_canonical_trainer_classes()
              after_count = len(after)
              patched = [(tf, cls.__name__) for tf, cls in after
                         if _is_unsloth_patched(cls)]
              unpatched = [(tf, cls.__name__) for tf, cls in after
                           if not _is_unsloth_patched(cls)]
              print(
                  f"\nCanonical trainer surface for TRL {trl.__version__}: "
                  f"discoverable_before={before_count} "
                  f"discoverable_after={after_count} "
                  f"patched={len(patched)} unpatched={len(unpatched)}"
              )
              for tf, n in patched:
                  print(f"  PATCHED   {tf}: {n}")
              for tf, n in unpatched:
                  print(f"  UNPATCHED {tf}: {n}")
              # Hard contract: every canonical trainer that imports
              # cleanly must end up Unsloth-prefixed after the umbrella
              # patch. If a trainer was discoverable BEFORE the patch but
              # is missing from `after`, that is a separate (rare) issue
              # we surface as failure.
              assert before_count == after_count, (
                  f"trainer-class set changed across patching: "
                  f"before={[n for _, n in before_unpatched]} "
                  f"after={[n for _, n in unpatched]}"
              )
              assert not unpatched, (
                  "unsloth.models.rl.patch_trl_rl_trainers did NOT patch: "
                  + ", ".join(f"{tf}:{n}" for tf, n in unpatched)
              )
              # Floor matches the cohort sizes from the TRL version sweep:
              # 18 (0.22-0.23), 15 (0.24-0.28), 6 (0.29+ canonical only).
              assert len(patched) >= 6, (
                  f"only {len(patched)} canonical trainers patched; "
                  "expected >= 6 (the smallest production cohort)."
              )


          def test_unsloth_patches_experimental_trainers_via_thin_wrappers():
              """TRL 0.29+ ships canonical-`trl.trainer.<x>_trainer` modules
              for many trainers as deprecation thin-wrappers that forward
              to `trl.experimental.<x>`. unsloth's
              `_patch_trl_rl_trainers` (rl.py:677-702) detects
              `trl.experimental` in the trainer source and resolves to
              the parent class -- so patching the canonical entry should
              also Unsloth-prefix the experimental class via in-module
              setattr.

              Verify by walking trl.experimental.* AST for every *Trainer
              class, then checking whether it (or any class with the same
              name in the experimental package) carries the Unsloth
              prefix after the umbrella patch."""
              from unsloth.models.rl import patch_trl_rl_trainers
              patch_trl_rl_trainers()
              experimental_pkgs = _enumerate_experimental_trainer_packages()
              if not experimental_pkgs:
                  pytest.skip(
                      f"TRL {trl.__version__} has no trl.experimental.* "
                      "trainer surface (pre-0.29 cohort). The canonical "
                      "test above already covers patching here."
                  )
              found = []
              missing = []
              for pkg_name, class_names in experimental_pkgs:
                  qual = f"trl.experimental.{pkg_name}"
                  try:
                      pkg_mod = importlib.import_module(qual)
                  except Exception as e:
                      # Optional-dep ImportError: experimental package
                      # could not be loaded. Match unsloth's runtime
                      # tolerance: this would also be silently skipped
                      # by `_patch_trl_rl_trainers`. Record but do not
                      # fail.
                      print(
                          f"  IMPORT-SKIP {qual}: "
                          f"{type(e).__name__}: {str(e)[:120]}"
                      )
                      continue
                  for cls_name in class_names:
                      cls = getattr(pkg_mod, cls_name, None)
                      if cls is None:
                          # Class is defined inside the package but not
                          # re-exported on the package init. Walk
                          # submodules to find it.
                          import pkgutil as _pku
                          for sub in _pku.walk_packages(
                              pkg_mod.__path__, prefix=qual + "."
                          ):
                              try:
                                  sub_mod = importlib.import_module(sub.name)
                              except Exception:
                                  continue
                              cls = getattr(sub_mod, cls_name, None)
                              if cls is not None:
                                  break
                      if cls is None:
                          missing.append((pkg_name, cls_name))
                          continue
                      if _is_unsloth_patched(cls):
                          found.append((pkg_name, cls_name))
                          print(f"  PATCHED   trl.experimental.{pkg_name}.{cls_name}")
                      else:
                          # Not Unsloth-prefixed: either unsloth chose
                          # not to patch this surface (e.g. the canonical
                          # thin-wrapper module did not exist) or the
                          # patch silently failed. Record both
                          # outcomes; the assertion below tolerates the
                          # gap as informational, not failure -- the
                          # canonical test enforces the hard contract.
                          print(
                              f"  NOT-PATCHED trl.experimental.{pkg_name}."
                              f"{cls_name} (no Unsloth-prefix on the "
                              "experimental surface)"
                          )
              total_experimental = sum(len(cs) for _, cs in experimental_pkgs)
              print(
                  f"\nExperimental trainer surface (TRL {trl.__version__}): "
                  f"{len(experimental_pkgs)} packages, "
                  f"{total_experimental} *Trainer classes; "
                  f"unsloth-patched={len(found)} class-missing={len(missing)}"
              )
              # Hard contract: a *Trainer class declared in a python
              # source file must be locatable in its package after import.
              # If we saw the class definition but cannot find the symbol
              # at runtime, the package's public surface drifted.
              assert not missing, (
                  "experimental *Trainer classes declared in source but "
                  f"not importable: {missing}"
              )
          PY
          python -m pytest -q --tb=short -s tests/_trl_trainer_discovery_shim.py
          rm -f tests/_trl_trainer_discovery_shim.py

      - name: MoE per-family coverage + GRPO patches + grouped_gemm AST
        # Catches the recurring class of bugs that PR #624 (gemma4 missing
        # extractor), PR #612 (gemma4 GRPO patch silently dropped), PR #607
        # (gate_up LoRA dropped from grad graph), PR #601 (qwen MoE shape
        # mismatch), unsloth#4934 (TRL disable_gradient_checkpointing
        # corrupts unsloth GC), and unsloth#3598 (gradient_accumulation
        # double-scale on accepts_loss_kwargs=False) targeted. Coverage:
        #
        #   1. Per-MoE-family side-effect contract: for every patch_*_moe
        #      function in unsloth_zoo.temporary_patches, if its target
        #      transformers class is importable on this matrix cell, the
        #      patch must mark the class with `_unsloth_already_patched=True`
        #      after running. This is exactly what unsloth_zoo's existing
        #      test_moe_lora_extractor_coverage walks at the registration
        #      level; here we tie each patch fn to its declared target so a
        #      silent early-return (PR #612 style) surfaces as red rather
        #      than a coverage skip.
        #
        #   2. PR #4934 (GRPO + TRL 1.0): patch_trl_disable_gradient_checkpointing
        #      must rebind trl.models.utils.disable_gradient_checkpointing to
        #      the unsloth no-op AND propagate the rebinding to every trl.*
        #      module that imported the symbol by reference.
        #
        #   3. PR #3598 (gradient_accumulation): patch_gradient_accumulation_fix
        #      must run cleanly on a synthetic Trainer whose training_step
        #      signature carries `num_items_in_batch`. The original bug was
        #      that `accepts_loss_kwargs=False` (Qwen3VL, Gemma3 in t-4.57)
        #      caused double loss-scaling; here we verify the rewrite path
        #      itself does not raise on a CPU-resolvable shape.
        #
        #   4. unsloth/kernels/moe/grouped_gemm AST smoke: the Triton kernels
        #      are GPU-only at runtime, but a SyntaxError or stray
        #      string-literal in the source still surfaces as a test-time
        #      ImportError on every install. ast.parse the .py files without
        #      executing.
        #
        # Wall-time per cell ~30-60s. Routed through pytest for the spoof
        # harness so unsloth_zoo.temporary_patches imports are clean.
        run: |
          set -euxo pipefail
          cat > tests/_moe_coverage_shim.py <<'PY'
          # Auto-generated by .github/workflows/consolidated-tests-ci.yml.
          import sys, pathlib, ast, importlib, importlib.util, contextlib, os
          sys.path.insert(0, str(pathlib.Path(__file__).parent))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()

          import pytest

          # Map each MoE patch function to the transformers classes it is
          # contractually responsible for marking with _unsloth_already_patched
          # after a successful run. Sourced from
          # unsloth_zoo/temporary_patches/<family>_moe.py:
          #   - qwen3_moe.py:382-398 patches Qwen3MoeExperts (new path) or
          #     Qwen3MoeSparseMoeBlock (old path).
          #   - qwen3_5_moe.py + qwen3_next_moe.py + qwen3_vl_moe.py register
          #     extractors on Qwen3_5MoeExperts / Qwen3NextExperts /
          #     Qwen3VLMoeTextExperts respectively.
          #   - gemma4_moe.py marks Gemma4TextExperts (current) or
          #     Gemma4TextMoEBlock (legacy).
          #   - glm4_moe.py marks Glm4MoeLiteNaiveMoe.
          #   - deepseek_v3_moe.py marks DeepseekV3NaiveMoe.
          #   - gpt_oss.py:patch_gpt_oss_moe_for_lora marks GptOssExperts.
          # Each cell skips a target if the transformers version lacks it
          # (legitimate version-skew); only patches with at least one
          # importable target are exercised.
          # Each entry = ((patch_module, patch_fn), targets, env_setup,
          # version_gate). env_setup runs before the patch fn (e.g. set
          # UNSLOTH_MODEL_NAME for gpt_oss). version_gate is a callable
          # returning True when the patch SHOULD run on this transformers;
          # if False, the test skips with a documented reason.
          def _v5_or_later():
              try:
                  import transformers
                  major = int(transformers.__version__.split(".")[0])
                  return major >= 5
              except Exception:
                  return False

          MOE_PATCHES = [
              {
                  "module": "unsloth_zoo.temporary_patches.qwen3_moe",
                  "fn": "patch_qwen3_moe",
                  "targets": [
                      ("transformers.models.qwen3_moe.modeling_qwen3_moe", "Qwen3MoeExperts"),
                      ("transformers.models.qwen3_moe.modeling_qwen3_moe", "Qwen3MoeSparseMoeBlock"),
                  ],
                  "env": {},
                  "gate": lambda: True,
                  "gate_reason": "",
              },
              {
                  "module": "unsloth_zoo.temporary_patches.qwen3_5_moe",
                  "fn": "patch_qwen3_5_moe",
                  "targets": [
                      ("transformers.models.qwen3_5_moe.modeling_qwen3_5_moe", "Qwen3_5MoeExperts"),
                  ],
                  "env": {}, "gate": lambda: True, "gate_reason": "",
              },
              {
                  "module": "unsloth_zoo.temporary_patches.qwen3_next_moe",
                  "fn": "patch_qwen3_next_moe",
                  "targets": [
                      ("transformers.models.qwen3_next.modeling_qwen3_next", "Qwen3NextExperts"),
                  ],
                  "env": {}, "gate": lambda: True, "gate_reason": "",
              },
              {
                  "module": "unsloth_zoo.temporary_patches.qwen3_vl_moe",
                  "fn": "patch_qwen3_vl_moe",
                  "targets": [
                      ("transformers.models.qwen3_vl_moe.modeling_qwen3_vl_moe", "Qwen3VLMoeTextExperts"),
                  ],
                  "env": {}, "gate": lambda: True, "gate_reason": "",
              },
              {
                  "module": "unsloth_zoo.temporary_patches.gemma4_moe",
                  "fn": "patch_gemma4_moe",
                  "targets": [
                      ("transformers.models.gemma4.modeling_gemma4", "Gemma4TextExperts"),
                  ],
                  "env": {}, "gate": lambda: True, "gate_reason": "",
              },
              {
                  "module": "unsloth_zoo.temporary_patches.glm4_moe",
                  "fn": "patch_glm4_moe",
                  "targets": [
                      ("transformers.models.glm4_moe.modeling_glm4_moe", "Glm4MoeLiteNaiveMoe"),
                  ],
                  "env": {}, "gate": lambda: True, "gate_reason": "",
              },
              {
                  "module": "unsloth_zoo.temporary_patches.deepseek_v3_moe",
                  "fn": "patch_deepseek_v3_moe",
                  "targets": [
                      ("transformers.models.deepseek_v3.modeling_deepseek_v3", "DeepseekV3NaiveMoe"),
                  ],
                  "env": {}, "gate": lambda: True, "gate_reason": "",
              },
              {
                  "module": "unsloth_zoo.temporary_patches.gpt_oss",
                  "fn": "patch_gpt_oss_moe_for_lora",
                  "targets": [
                      ("transformers.models.gpt_oss.modeling_gpt_oss", "GptOssExperts"),
                  ],
                  # The patch reads UNSLOTH_MODEL_NAME and only runs when
                  # "gpt_oss" is in the normalized form. Set it explicitly
                  # so the gate at gpt_oss.py:1387 passes; otherwise the
                  # patch silently early-returns and the test would
                  # spuriously fail.
                  "env": {"UNSLOTH_MODEL_NAME": "gpt_oss"},
                  # Additionally only runs on transformers >= 5
                  # (gpt_oss.py:1392 `_is_transformers_v5()` gate).
                  "gate": _v5_or_later,
                  "gate_reason": (
                      "patch_gpt_oss_moe_for_lora gates on "
                      "transformers >= 5 (split-LoRA grouped_mm path)"
                  ),
              },
          ]


          def _resolve_target_classes(targets):
              """Return [(qual, cls), ...] for every importable target."""
              out = []
              for mod_path, cls_name in targets:
                  try:
                      mod = importlib.import_module(mod_path)
                  except Exception:
                      continue
                  cls = getattr(mod, cls_name, None)
                  if cls is None:
                      continue
                  out.append((f"{mod_path}.{cls_name}", cls))
              return out


          @pytest.mark.parametrize(
              "spec",
              MOE_PATCHES,
              ids=lambda s: s["fn"],
          )
          def test_moe_patch_marks_its_target_when_class_present(spec, monkeypatch):
              """If at least one target class is importable AND the
              version gate passes, run the patch fn and assert at least
              one target is marked patched afterwards. Skips when the
              transformers version lacks every target or when the
              version gate blocks the patch (legitimate). Fails on
              silent patch-fn early-returns (PR #612 class of bug)."""
              targets = spec["targets"]
              patch_module = spec["module"]
              patch_name = spec["fn"]
              importable = _resolve_target_classes(targets)
              if not importable:
                  pytest.skip(
                      f"{patch_name}: no target class importable on this "
                      f"transformers (looked for {[c for _, c in targets]})."
                  )
              if not spec["gate"]():
                  pytest.skip(
                      f"{patch_name}: version gate blocks this cell. "
                      f"Reason: {spec['gate_reason']}"
                  )
              for k, v in spec["env"].items():
                  monkeypatch.setenv(k, v)
              try:
                  pmod = importlib.import_module(patch_module)
              except Exception as e:
                  pytest.skip(
                      f"{patch_module} import failed (likely optional dep): "
                      f"{type(e).__name__}: {e}"
                  )
              fn = getattr(pmod, patch_name, None)
              if fn is None or not callable(fn):
                  pytest.skip(f"{patch_module} has no callable {patch_name}")
              try:
                  fn()
              except Exception as e:
                  raise AssertionError(
                      f"{patch_name}() raised on a transformers that "
                      f"DOES ship at least one target class ({importable}). "
                      f"This is the silent-failure mode PR #612 fixed: "
                      f"{type(e).__name__}: {e}"
                  )
              # At least one importable target must now carry SOME marker
              # showing unsloth touched it. Accepted signals (each is set
              # by a different patch flow in unsloth_zoo):
              #   - `_unsloth_already_patched=True`            (gemma4, deepseek_v3, glm4)
              #   - `_unsloth_lora_patched=True`               (gpt_oss_moe_for_lora)
              #   - `_unsloth_lora_extractor_fn` is callable   (qwen3_*, glm4_moe)
              #   - `_original_<modeling_tail>_<ClassName>_forward` attr
              #     (set by patch_function: qwen3_moe SparseMoeBlock, etc.)
              #   - `_original_forward` attribute              (gpt_oss in-place patch)
              # Accept any one as "patched".
              def _is_patched(cls) -> bool:
                  if getattr(cls, "_unsloth_already_patched", False) is True:
                      return True
                  if getattr(cls, "_unsloth_lora_patched", False) is True:
                      return True
                  if callable(getattr(cls, "_unsloth_lora_extractor_fn", None)):
                      return True
                  if "_original_forward" in dir(cls):
                      return True
                  cls_name = cls.__name__
                  for attr in dir(cls):
                      if attr.startswith("_original_") and attr.endswith(
                          f"_{cls_name}_forward"
                      ):
                          return True
                  return False

              after = _resolve_target_classes(targets)
              marked = [qual for qual, cls in after if _is_patched(cls)]
              if not marked:
                  raise AssertionError(
                      f"{patch_name}() ran without exception but no target "
                      f"in {importable} carries any of the unsloth markers "
                      "(_unsloth_already_patched / _unsloth_lora_patched / "
                      "_unsloth_lora_extractor_fn / _original_*_forward). "
                      "Patch silently no-op'd (PR #612 class of bug)."
                  )
              print(f"  {patch_name}: marked {marked}")


          # ---- PR #4934 (TRL 1.0+ GRPO disable_gradient_checkpointing) ----

          def test_patch_trl_disable_gradient_checkpointing():
              """unsloth/models/rl.py:patch_trl_disable_gradient_checkpointing
              must rebind trl.models.utils.disable_gradient_checkpointing to
              the unsloth no-op when TRL >= 1.0. Pre-1.0 TRL has no such
              symbol -> the patch returns early."""
              try:
                  import trl.models.utils as _tmu
              except ImportError:
                  pytest.skip("trl not installed")
              had_symbol = hasattr(_tmu, "disable_gradient_checkpointing")
              try:
                  from unsloth.models.rl import patch_trl_disable_gradient_checkpointing
              except ImportError:
                  pytest.skip(
                      "unsloth.models.rl.patch_trl_disable_gradient_checkpointing "
                      "absent (older unsloth than #4934)"
                  )
              patch_trl_disable_gradient_checkpointing()
              if not had_symbol:
                  # Pre-1.0 TRL: patch is a no-op early-return. Verify
                  # nothing broke.
                  pytest.skip(
                      "TRL pre-1.0 has no disable_gradient_checkpointing; "
                      "patch correctly early-returned."
                  )
              fn = getattr(_tmu, "disable_gradient_checkpointing", None)
              assert fn is not None, (
                  "trl.models.utils.disable_gradient_checkpointing missing "
                  "after patch -- patch removed the symbol entirely?"
              )
              assert getattr(fn, "_unsloth_noop_patched", False) is True, (
                  "trl.models.utils.disable_gradient_checkpointing was NOT "
                  "rebound to the unsloth no-op. PR #4934 regression."
              )
              # PR #4934 also walks sys.modules to rebind trl.* modules
              # that imported the symbol by reference. Verify at least the
              # canonical trainer modules picked up the rebinding when
              # they re-export it.
              import sys
              checked = 0
              missed = []
              for mod_name, mod in list(sys.modules.items()):
                  if not mod_name.startswith("trl."):
                      continue
                  bound = getattr(mod, "disable_gradient_checkpointing", None)
                  if bound is None:
                      continue
                  checked += 1
                  if not getattr(bound, "_unsloth_noop_patched", False):
                      missed.append(mod_name)
              print(f"  rebound disable_gradient_checkpointing in {checked} trl.* modules")
              assert not missed, (
                  "trl.* modules that imported disable_gradient_checkpointing "
                  f"by reference but did not get rebound: {missed}"
              )


          # ---- PR #3598 (gradient_accumulation loss-scaling rewrite) ----

          def test_patch_gradient_accumulation_fix_runs_on_synthetic_trainer():
              """patch_gradient_accumulation_fix rewrites a Trainer's
              `training_step` source via inspect+exec when the signature
              carries `num_items_in_batch`. PR #3598 fixed the rewrite
              path to not double-scale for trainers with
              `accepts_loss_kwargs=False`. Verify the patch fn runs
              without raising on a synthetic Trainer carrying that
              signature."""
              try:
                  from unsloth.models._utils import patch_gradient_accumulation_fix
              except ImportError:
                  pytest.skip(
                      "unsloth.models._utils.patch_gradient_accumulation_fix absent"
                  )
              try:
                  from transformers import Trainer
              except ImportError:
                  pytest.skip("transformers.Trainer absent")
              # The patch reads the live Trainer.training_step source. We
              # exercise the standard transformers.Trainer here -- if the
              # bug is reintroduced in the source rewriter (e.g. broken
              # exec, missing import injection), the patch fn raises.
              try:
                  patch_gradient_accumulation_fix(Trainer)
              except Exception as e:
                  raise AssertionError(
                      "patch_gradient_accumulation_fix raised on a vanilla "
                      f"transformers.Trainer: {type(e).__name__}: {e}"
                  )
              # Idempotency: second call must not raise either (the rewrite
              # adds `_unsloth_training_step` marker so the second call
              # short-circuits per _utils.py:1692-1693).
              patch_gradient_accumulation_fix(Trainer)


          # ---- unsloth/kernels/moe/grouped_gemm AST smoke ----

          def _walk_py_files(root: pathlib.Path):
              for p in root.rglob("*.py"):
                  if "__pycache__" in p.parts:
                      continue
                  yield p


          def test_unsloth_kernels_moe_grouped_gemm_ast_parses():
              """unsloth/kernels/moe/grouped_gemm hosts the Triton MoE
              kernels (GPU-only at runtime). A SyntaxError or stray token
              at the SOURCE level still surfaces as ImportError on every
              install, so AST-parse the .py files without executing."""
              # Locate `unsloth/kernels/moe/grouped_gemm` via the installed
              # `unsloth` package.
              import unsloth as _unsloth
              kernel_root = (
                  pathlib.Path(_unsloth.__file__).parent
                  / "kernels" / "moe" / "grouped_gemm"
              )
              if not kernel_root.exists():
                  pytest.skip(
                      f"{kernel_root} not present in this unsloth checkout."
                  )
              fail = []
              ok = 0
              for p in _walk_py_files(kernel_root):
                  try:
                      ast.parse(p.read_text(encoding="utf-8"), filename=str(p))
                      ok += 1
                  except SyntaxError as e:
                      fail.append((str(p), f"SyntaxError: {e}"))
                  except Exception as e:
                      fail.append((str(p), f"{type(e).__name__}: {e}"))
              print(f"AST-parsed {ok} grouped_gemm files; failed={len(fail)}")
              for path, err in fail:
                  print(f"  AST FAIL {path}: {err}")
              assert not fail, (
                  f"AST parse failed for {len(fail)} grouped_gemm files"
              )
              # Sanity: the directory MUST contain at least the interface
              # + kernels + reference subtrees as documented.
              expected = [
                  "interface.py",
                  "kernels/forward.py",
                  "kernels/backward.py",
                  "reference/moe_block.py",
                  "reference/moe_ops.py",
              ]
              missing = [e for e in expected if not (kernel_root / e).is_file()]
              assert not missing, (
                  "grouped_gemm directory layout regressed; missing: "
                  f"{missing}"
              )
          PY
          python -m pytest -q --tb=short -s tests/_moe_coverage_shim.py
          rm -f tests/_moe_coverage_shim.py

      - name: Summary
        if: always()
        run: |
          echo "::group::Versions"
          python -c "import sys, platform; print(sys.version); print(platform.platform())"
          python -c "import torch; print('torch', torch.__version__, 'cuda?', torch.cuda.is_available())"
          python -c "import transformers; print('transformers', transformers.__version__)"
          # `pip show` instead of `import unsloth_zoo` — its __init__ raises
          # without an accelerator and the spoof harness only kicks in under
          # pytest. Cheap and accurate.
          pip show unsloth_zoo
          echo "::endgroup::"
          echo "Consolidated job done. Coverage:"
          echo "  - 16 unsloth Bucket-A tests under tests/saving/ + tests/utils/"
          echo "  - unsloth_zoo @ ${UNSLOTH_ZOO_REF} pytest tests/ (5 GPU cases deselected)"
          echo "  - unsloth_zoo.compiler.test_apply_fused_lm_head"

  llama-cpp-smoke:
    # Standalone llama.cpp build + smoke. Earlier this lived inside every
    # consolidated matrix cell and re-cmake'd llama.cpp ~5 min per cell --
    # 3 cells x 275 s = ~14 min of duplicated CPU on every PR for an
    # artefact that has nothing to do with the (transformers, TRL) combo.
    # `install_llama_cpp` clones ggml-org/llama.cpp at a pinned commit and
    # builds the LLAMA_CPP_TARGETS list; the result is independent of the
    # HF stack version. Run once, gate the PR.
    name: llama.cpp build + smoke
    runs-on: ubuntu-latest
    timeout-minutes: 25
    env:
      UNSLOTH_ZOO_REF: ${{ inputs.unsloth_zoo_ref || 'main' }}
      # Same env contract the matrix cells use: protobuf python parser
      # (transformers' bundled *_pb2.py needs it), studio on PYTHONPATH,
      # compile-disable + UNSLOTH_IS_PRESENT so unsloth_zoo's __init__
      # bootstrap accepts a pure-import.
      PROTOCOL_BUFFERS_PYTHON_IMPLEMENTATION: python
      PYTHONPATH: ${{ github.workspace }}/studio
      UNSLOTH_COMPILE_DISABLE: '1'
      UNSLOTH_IS_PRESENT: '1'
    steps:
      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
        with:
          persist-credentials: false

      - uses: actions/setup-python@a309ff8b426b58ec0e2a45f0f869d46889d02405  # v6.2.0
        with:
          python-version: '3.12'
          cache: 'pip'

      - name: Install runtime deps for unsloth_zoo.llama_cpp
        # unsloth_zoo's `__init__` imports `temporary_patches`, which
        # in turn pulls per-architecture submodules (gemma3n, gemma4,
        # qwen3_*_moe, glm4_moe, deepseek_v3_moe, pixtral, ministral,
        # mxfp4, bitsandbytes, flex_attention_bwd) -- many of those
        # transitively touch transformers and peft / accelerate. Mirror
        # the matrix job's install minus the heavy bits that have no
        # bearing on `install_llama_cpp` itself: studio.txt's FastAPI
        # stack, bitsandbytes (CUDA-only build dependency), triton,
        # mammoth/unpdf (PDF tools), datasets, sqlalchemy/cryptography,
        # pytest (we run no tests). The remaining pin shape matches
        # studio-backend-ci.yml's "Repo tests (CPU)" baseline.
        run: |
          set -euxo pipefail
          python -m pip install --upgrade pip
          # Match the matrix job's torch path so unsloth_zoo's
          # `import torch` resolves to the same CPU build.
          pip install --index-url https://download.pytorch.org/whl/cpu --extra-index-url https://pypi.org/simple \
            'torch>=2.4,<2.11' 'torchvision<0.26'
          pip install \
            'numpy<3' protobuf sentencepiece \
            requests tqdm psutil packaging safetensors \
            'peft>=0.18,<0.20' 'accelerate>=0.34,<2'
          # transformers + trl come from pyproject.toml's pinned line
          # so this job stays in sync with whatever the consolidated
          # `__from_pyproject__` matrix cell is using.
          pip install transformers trl
          pip install -e . --no-deps

      - name: Clone unsloth_zoo @ ${{ env.UNSLOTH_ZOO_REF }}
        # Same shallow clone as the matrix job; we install editable so
        # `unsloth_zoo.llama_cpp` resolves to the cloned tree (and any
        # main-branch fixes flow into the smoke without a release).
        run: |
          set -euxo pipefail
          # github.com occasionally 500s on the git fetch; retry so a
          # single upstream blip does not fail CI.
          for attempt in 1 2 3; do
            rm -rf "$RUNNER_TEMP/unsloth-zoo"
            if git clone --depth=1 --branch="$UNSLOTH_ZOO_REF" \
                https://github.com/unslothai/unsloth-zoo \
                "$RUNNER_TEMP/unsloth-zoo"; then
              break
            fi
            if [ "$attempt" -eq 3 ]; then
              echo "::error::git clone unsloth-zoo failed after 3 attempts"
              exit 1
            fi
            delay=$((5 * attempt))
            echo "::warning::clone failed (attempt $attempt/3), retrying in ${delay}s..."
            sleep "$delay"
          done
          pip install -e "$RUNNER_TEMP/unsloth-zoo" --no-deps
          pip show unsloth_zoo

      - name: llama.cpp install via unsloth_zoo.llama_cpp + CLI `--help` smoke
        # Exercise the canonical `unsloth_zoo.llama_cpp.install_llama_cpp`
        # flow that GGUF export uses at runtime: clone ggml-org/llama.cpp
        # into ~/.unsloth/llama.cpp, build the LLAMA_CPP_TARGETS list
        # (llama-quantize, llama-cli, llama-mtmd-cli, llama-gguf-split,
        # llama-server) via cmake, then run `--help` on whichever CLI
        # inference binary the build actually produced.
        #
        # This replaces the previous "download upstream prebuilt zip"
        # approach, which silently exited 0 with the message
        # "no ubuntu-x64 prebuilt asset" when ggml-org's release-asset
        # naming drifted (the regex `bin-ubuntu-x64.*\.zip$` no longer
        # matched their current asset names). The build path is the same
        # one Unsloth users hit in production via `model.save_pretrained_gguf`.
        #
        # We do NOT hard-require `llama-cli` specifically: upstream
        # ggml-org/llama.cpp moved the cli/server/ui targets behind the
        # `LLAMA_BUILD_SERVER` cmake option (tools/CMakeLists.txt) and the
        # set of binaries that survive a given checkout drifts over time
        # (e.g. a recent build root shipped llama-server + llama-quantize
        # + llama-diffusion-cli but no llama-cli). The durable contract is
        # "install_llama_cpp produced a working CLI inference binary AND a
        # working quantizer", so we --help-probe the first of
        # llama-cli / llama-mtmd-cli / llama-server that exists. If a
        # future llama.cpp restores llama-cli it is first in the list and
        # is preferred, so this stays backwards compatible.
        #
        # Wall-time budget: ~3-5 min cold, dominated by cmake build of
        # 5 targets on the runner's 4 cores. Apt-package install is
        # handled by `install_llama_cpp` itself via its
        # `check_build_requirements` -> `install_package` chain.
        run: |
          set -euxo pipefail
          # libssl-dev / libcurl4-openssl-dev are needed by llama.cpp's
          # cmake build for HTTPS support; install up-front so the
          # `install_llama_cpp` requirement-check is a no-op.
          sudo apt-get update -qq
          sudo apt-get install -y -qq build-essential cmake git curl \
            libgomp1 libssl-dev libcurl4-openssl-dev
          python <<'PY'
          import os, shutil, subprocess, sys, pathlib
          # Apply the same CPU spoof the pytest shims use BEFORE any
          # unsloth_zoo import: unsloth_zoo/__init__.py calls
          # device_type.get_device_type() at module load and raises
          # `NotImplementedError: Unsloth cannot find any torch
          # accelerator` on a GPU-less runner. The spoof flips
          # torch.cuda.is_available() to True so the device probe takes
          # the cuda branch; we never actually run CUDA tensor ops in
          # this step (just clone+cmake+--help on the binaries).
          sys.path.insert(0, str(pathlib.Path("tests").resolve()))
          import _zoo_aggressive_cuda_spoof as _spoof
          _spoof.apply()
          from unsloth_zoo.llama_cpp import (
              install_llama_cpp,
              LLAMA_CPP_DEFAULT_DIR,
              LLAMA_CPP_TARGETS,
          )
          print(f"Unsloth llama.cpp default dir: {LLAMA_CPP_DEFAULT_DIR}")
          print(f"Build targets: {LLAMA_CPP_TARGETS}")
          # install_llama_cpp returns (quantizer_path, converter_script_path).
          # The quantizer's directory is the `llama.cpp` install root, which
          # also holds the CLI inference binaries after build/bin/llama-* gets
          # copied up (llama_cpp.py:1450-1454; on Windows they stay in
          # build/bin/Release/).
          quantizer, converter = install_llama_cpp(print_output=True)
          assert quantizer and os.path.exists(quantizer), (
              f"install_llama_cpp returned quantizer={quantizer!r} but file missing"
          )
          assert converter and os.path.isfile(converter), (
              f"install_llama_cpp returned converter={converter!r} but missing"
          )
          install_root = os.path.dirname(quantizer)
          is_windows = sys.platform == "win32"
          exe = ".exe" if is_windows else ""
          # Search both the copied-up root and the Windows build/bin/Release/
          # location the quantizer might already live in.
          search_dirs = [install_root]
          win_release = os.path.join(install_root, "build", "bin", "Release")
          if win_release not in search_dirs:
              search_dirs.append(win_release)
          # Any of these proves a working llama.cpp CLI inference binary was
          # built. Order = preference: llama-cli is canonical (restored first
          # if upstream brings it back), then the multimodal CLI, then the
          # server (always built whenever cli would be, behind LLAMA_BUILD_SERVER).
          cli_names = [f"llama-cli{exe}", f"llama-mtmd-cli{exe}", f"llama-server{exe}"]
          cli = None
          cli_name = None
          for name in cli_names:
              for d in search_dirs:
                  candidate = os.path.join(d, name)
                  if os.path.exists(candidate) and (is_windows or os.access(candidate, os.X_OK)):
                      cli, cli_name = candidate, name
                      break
              if cli is not None:
                  break
          if cli is None:
              found = []
              for d in search_dirs:
                  if os.path.isdir(d):
                      found += [p for p in os.listdir(d) if p.startswith("llama-")]
              raise AssertionError(
                  f"No CLI inference binary ({', '.join(cli_names)}) found after "
                  f"build in {search_dirs}. Build root contents: {sorted(set(found))[:20]}"
              )
          print(f"Using CLI inference binary: {cli_name} -> {cli}")
          # `--help` exits non-zero on some builds; the contract is that
          # recognizable help text appears on stdout/stderr. llama-server
          # exposes a different flag set than llama-cli, so accept its
          # tokens too (e.g. --host / --port / "server").
          proc = subprocess.run(
              [cli, "--help"], capture_output=True, text=True, timeout=30,
          )
          combined = (proc.stdout or "") + (proc.stderr or "")
          print(f"--- {cli_name} --help (first 30 lines) ---")
          print("\n".join(combined.splitlines()[:30]))
          assert any(
              tok in combined.lower()
              for tok in ("usage", "--help", "--model", "-m,", "--host", "--port", "server")
          ), (
              f"{cli_name} --help produced no recognizable help text. "
              f"exit={proc.returncode}\nstdout: {proc.stdout[:400]!r}\n"
              f"stderr: {proc.stderr[:400]!r}"
          )
          # Also exercise the quantizer the way GGUF export does: --help
          # round-trip on the binary that does the actual heavy lifting.
          q = subprocess.run(
              [quantizer, "--help"], capture_output=True, text=True, timeout=15,
          )
          q_combined = (q.stdout or "") + (q.stderr or "")
          assert "usage" in q_combined.lower() or "type" in q_combined.lower(), (
              f"llama-quantize --help produced no help text. "
              f"exit={q.returncode}\nstdout: {q.stdout[:400]!r}\n"
              f"stderr: {q.stderr[:400]!r}"
          )
          print(
              f"\nOK: install_llama_cpp produced a working {cli_name} at {cli} "
              f"and llama-quantize at {quantizer}."
          )
          PY