chore: import upstream snapshot with attribution

scripts/ci_clean_cache.py

@@ -0,0 +1,68 @@
+"""
+Utility to clean cache files that exceed a specific time in days according to their
+last access time recorded in the cache.
+
+Exit code:
+- 1 if no candidates are found
+- 0 if candidates are found
+
+Deletion can be enabled by passing `-d` parameter, otherwise it will only list the candidates.
+"""
+
+import sys
+from datetime import datetime as dt
+from datetime import timezone
+
+from huggingface_hub import scan_cache_dir
+
+
+def find_old_revisions(scan_results, max_age_days=30):
+    """Find commit hashes of objects in the cache. These objects need a last access time that
+    is above the passed `max_age_days` parameter. Returns an empty list if no objects are found.
+    Time measurement is based of the current time and the recorded last access tiem in the cache.
+    """
+    now = dt.now(timezone.utc)
+    revisions = [(i.revisions, i.last_accessed) for i in scan_results.repos]
+    revisions_ages = [(rev, (now - dt.fromtimestamp(ts_access, timezone.utc)).days) for rev, ts_access in revisions]
+    delete_candidates = [rev for rev, age in revisions_ages if age > max_age_days]
+    hashes = [n.commit_hash for rev in delete_candidates for n in rev]
+
+    return hashes
+
+
+def delete_old_revisions(scan_results, delete_candidates, do_delete=False):
+    delete_operation = scan_results.delete_revisions(*delete_candidates)
+    print(f"Would free {delete_operation.expected_freed_size_str}")
+    print(f"Candidates: {delete_candidates}")
+
+    if do_delete:
+        print("Deleting now.")
+        delete_operation.execute()
+    else:
+        print("Not deleting, pass the -d flag.")
+
+
+if __name__ == "__main__":
+    from argparse import ArgumentParser
+
+    parser = ArgumentParser()
+    parser.add_argument("-a", "--max-age", type=int, default=30, help="Max. age in days items in the cache may have.")
+    parser.add_argument(
+        "-d",
+        "--delete",
+        action="store_true",
+        help=(
+            "Delete mode; Really delete items if there are candidates. Exit code = 0 when we found something to delete, 1 "
+            "otherwise."
+        ),
+    )
+    args = parser.parse_args()
+
+    scan_results = scan_cache_dir()
+
+    delete_candidates = find_old_revisions(scan_results, args.max_age)
+    if not delete_candidates:
+        print("No delete candidates found, not deleting anything.")
+        sys.exit(1)
+
+    delete_old_revisions(scan_results, delete_candidates, do_delete=args.delete)

scripts/convert-bone-to-miss.py

@@ -0,0 +1,69 @@
+# Copyright (c) 2025 Your Organization/Project. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Convert Bone checkpoint to MiSS format."""
+
+import argparse
+import json
+import os
+from pathlib import Path
+
+from safetensors import safe_open
+from safetensors.torch import save_file
+
+from peft.utils import CONFIG_NAME, SAFETENSORS_WEIGHTS_NAME
+
+
+def convert_bone_to_miss(bone_dir: Path, miss_dir: Path) -> None:
+    """Convert Bone checkpoint files to MiSS format."""
+    bone_config_path = bone_dir / CONFIG_NAME
+    miss_config_path = miss_dir / CONFIG_NAME
+    if not os.path.exists(miss_dir):
+        os.makedirs(miss_dir, exist_ok=True)
+    with open(bone_config_path, encoding="utf-8") as f:
+        config = json.load(f)
+
+    config["peft_type"] = "MISS"
+
+    with open(miss_config_path, "w", encoding="utf-8") as f:
+        json.dump(config, f, indent=2, ensure_ascii=False)
+
+    bone_weight_path = bone_dir / SAFETENSORS_WEIGHTS_NAME
+    miss_weight_path = miss_dir / SAFETENSORS_WEIGHTS_NAME
+
+    new_data = {}
+
+    with safe_open(bone_weight_path, framework="pt") as f:
+        for old_key in f.keys():
+            tensor = f.get_tensor(old_key)
+            new_key = old_key.replace(".bone_", ".miss_")
+            new_data[new_key] = tensor
+
+    save_file(new_data, miss_weight_path)
+
+    print(f"Converted checkpoint saved at {miss_weight_path}")
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Convert Bone checkpoint to MiSS format.")
+    parser.add_argument("bone_dir", type=Path, help="Directory containing Bone checkpoint files")
+    parser.add_argument("miss_dir", type=Path, help="Directory to save MiSS checkpoint files")
+    args = parser.parse_args()
+
+    args.miss_dir.mkdir(parents=True, exist_ok=True)
+    convert_bone_to_miss(args.bone_dir, args.miss_dir)
+
+
+if __name__ == "__main__":
+    main()

scripts/evaluate-lora-conversion.py

@@ -0,0 +1,156 @@
+# Copyright 2025-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""Script to evaluate a PEFT checkpoint converted into a LoRA on GSM8K
+
+To run this script, first train a PEFT model on MetaMathQA as described here:
+
+https://github.com/huggingface/peft/tree/main/method_comparison/MetaMathQA
+
+Call the script with the `-v` (verbose) option. When that run finishes, it will save a checkpoint of that model and
+print a message like this: "Saved PEFT checkpoint to ...". Use this path as the `--path` argument to this script.
+
+Example usage:
+
+```bash
+# Convert to LoRA with rank 8 and evaluate it
+python evaluate-lora-conversion.py --path /path/to/peft/checkpoint --rank 8
+# Convert to LoRA with dynamic rank (50% singular value threshold) and evaluate it
+python evaluate-lora-conversion.py --path /path/to/peft/checkpoint --rank 0.5
+# Evaluate the original PEFT model without LoRA conversion
+python evaluate-lora-conversion.py --path /path/to/peft/checkpoint
+```
+
+The script will report the evaluation accuracy, maximum CUDA memory reserved, and evaluation time for the converted LoRA
+model.
+
+"""
+
+import argparse
+import importlib.util
+import os
+import sys
+import time
+
+import torch
+from transformers import AutoModelForCausalLM
+
+from peft import PeftModel, convert_to_lora, get_peft_model, set_peft_model_state_dict
+
+
+root = os.path.abspath(os.path.join(os.path.dirname(__file__), ".."))
+
+spec = importlib.util.spec_from_file_location("data", os.path.join(root, "method_comparison", "MetaMathQA", "data.py"))
+mm_data = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mm_data)
+sys.modules["data"] = mm_data
+
+spec = importlib.util.spec_from_file_location(
+    "utils", os.path.join(root, "method_comparison", "MetaMathQA", "utils.py")
+)
+mm_utils = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mm_utils)
+sys.modules["utils"] = mm_utils
+
+spec = importlib.util.spec_from_file_location("run", os.path.join(root, "method_comparison", "MetaMathQA", "run.py"))
+mm_run = importlib.util.module_from_spec(spec)
+spec.loader.exec_module(mm_run)
+
+
+def noop(*args, **kwargs):
+    pass
+
+
+def evaluate_model(model, tokenizer, ds_test):
+    torch.cuda.empty_cache()
+    torch.cuda.reset_peak_memory_stats()
+    tic = time.perf_counter()
+    predictions, responses = mm_run.evaluate(
+        model=model,
+        tokenizer=tokenizer,
+        ds=ds_test,
+        batch_size=50,
+        generate_kwargs={"max_length": 800, "max_new_tokens": 300, "pad_token_id": tokenizer.eos_token_id},
+        use_tqdm=True,
+    )
+    toc = time.perf_counter()
+    accuracy_peft = mm_utils.get_accuracy(predictions=predictions, responses=responses)
+    cuda_mem_reserved_max = torch.cuda.memory_reserved(0)
+    print(f"Evaluation Accuracy: {100 * accuracy_peft:.2f}%")
+    print(f"Max CUDA Memory Reserved: {cuda_mem_reserved_max / (1024**3):.2f} GB")
+    print(f"Evaluation Time: {toc - tic:.0f} seconds".format(toc - tic))
+
+
+def main(path_peft_model: str, rank: float | None) -> None:
+    model_id = "meta-llama/Llama-3.2-3B"
+    tokenizer = mm_utils.get_tokenizer(model_id=model_id, max_seq_length=768)
+    _, _, ds_test = mm_data.get_train_valid_test_datasets(
+        tokenizer=tokenizer, query_template="Question: {query} Think step by step.\nAnswer:", print_fn=noop
+    )
+
+    model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.bfloat16).to(0)
+    model = PeftModel.from_pretrained(model, path_peft_model)
+    if rank is None:
+        print("Evaluating the original PEFT model without LoRA conversion...")
+        model.set_adapter("default")
+        model.print_trainable_parameters()
+        model.eval()
+        evaluate_model(model, tokenizer, ds_test)
+        return
+
+    print(f"Converting PEFT model to LoRA with rank={rank}...")
+    tic = time.perf_counter()
+    lora_config, lora_state_dict = convert_to_lora(model, rank=rank, progressbar=True)
+    toc = time.perf_counter()
+    print(f"Conversion completed in {toc - tic:.0f} seconds.".format(toc - tic))
+
+    del model
+    torch.cuda.empty_cache()
+    model = AutoModelForCausalLM.from_pretrained(model_id, dtype=torch.bfloat16).to(0)
+
+    model = get_peft_model(model, lora_config)
+    model.print_trainable_parameters()
+
+    load_result = set_peft_model_state_dict(model, lora_state_dict)
+    assert not load_result.unexpected_keys, (
+        f"Unexpected keys when loading LoRA state dict: {load_result.unexpected_keys}"
+    )
+
+    del lora_state_dict
+    model.eval()
+    evaluate_model(model, tokenizer, ds_test)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(description="Evaluate a PEFT checkpoint converted into a LoRA on GSM8K")
+    parser.add_argument(
+        "--path",
+        type=str,
+        required=True,
+        help="Path to the input PEFT checkpoint",
+    )
+    parser.add_argument(
+        "--rank",
+        required=False,
+        default=None,
+        help="Rank for the LoRA decomposition (int, float, or None for no conversion)",
+    )
+
+    args = parser.parse_args()
+    if args.rank is not None:
+        if "." in str(args.rank):
+            args.rank = float(args.rank)
+        else:
+            args.rank = int(args.rank)
+    main(args.path, args.rank)

scripts/generate_method_capabilities.py

@@ -0,0 +1,769 @@
+# Copyright 2026-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""Generate a machine-readable capability matrix of all PEFT methods.
+
+For each registered PEFT method, a fixed set of checks ("tasks") determines which user-facing features the method
+supports, e.g. which quantization backends it integrates with, which layer types it can target, or whether its adapters
+can be merged into the base weights. The result is written as JSON and is intended as a generic data source, e.g. for
+documentation pages or the PEFT shop app (method_comparison/peft-shop).
+
+Each value is annotated with the *source* of the information:
+
+- "introspection": determined by statically inspecting the classes of the installed PEFT package
+- "file_check":    determined from the presence of integration modules in the PEFT source tree
+- "probe":         determined empirically by exercising the feature on a tiny model on CPU
+- "error":         the check itself failed; the value is "unknown" and the note contains the reason
+
+Values are never guessed: if a check cannot determine a feature, the value is reported as "unknown" together with a
+note. In particular, probing a method requires that its config can be instantiated with default arguments; methods that
+need more than that must have an entry in PROBE_CONFIG_OVERRIDES.
+
+The script requires PEFT to be installed (e.g. `pip install -e .`), runs on CPU, downloads nothing, and is idempotent:
+running it twice on the same environment produces identical output.
+
+Usage examples:
+
+    # check all methods, write JSON to method_capabilities.json
+    python scripts/generate_method_capabilities.py
+
+    # check only LoRA and IA3, write to a custom file
+    python scripts/generate_method_capabilities.py --methods lora ia3 --output capabilities.json
+
+    # show which checks would run, without running them
+    python scripts/generate_method_capabilities.py --dry-run
+
+Tasks are collected up-front before any of them runs (see `collect_tasks`). This makes `--dry-run` trivial and leaves
+the door open to run independent tasks in parallel later, should runtime ever become an issue.
+"""
+
+import argparse
+import dataclasses
+import enum
+import inspect
+import json
+import logging
+import re
+import sys
+import warnings
+from abc import ABC, abstractmethod
+from collections.abc import Callable
+from copy import deepcopy
+from dataclasses import dataclass
+from pathlib import Path
+from typing import Any, ClassVar
+
+import torch
+from torch import nn
+from tqdm import tqdm
+from transformers import LlamaConfig, LlamaForCausalLM
+from transformers.pytorch_utils import Conv1D
+
+import peft
+from peft import get_peft_model
+from peft.config import PeftConfig, PromptLearningConfig
+from peft.mapping import PEFT_TYPE_TO_CONFIG_MAPPING, PEFT_TYPE_TO_MIXED_MODEL_MAPPING, PEFT_TYPE_TO_TUNER_MAPPING
+from peft.tuners.tuners_utils import BaseTuner, BaseTunerLayer
+from peft.utils.hotswap import CONFIG_KEYS_TO_CHECK
+from peft.utils.peft_types import PeftType
+
+
+logger = logging.getLogger("generate_method_capabilities")
+
+UNKNOWN = "unknown"
+NOT_APPLICABLE = "not_applicable"
+
+# Probe models use this hidden dimension throughout. It is chosen to be highly divisible, since several methods have
+# divisibility constraints between their block/rank settings and the layer dimensions (e.g. C3A, VBLoRA, RoAd).
+HIDDEN_DIM = 64
+
+# Quantization integration modules inside a tuner package, mapped to the backend names they provide. The bnb module
+# always contains both the 8-bit and the 4-bit integration.
+QUANT_FILE_BACKENDS: dict[str, tuple[str, ...]] = {
+    "aqlm": ("aqlm",),
+    "awq": ("awq",),
+    "bnb": ("bnb_8bit", "bnb_4bit"),
+    "eetq": ("eetq",),
+    "gptq": ("gptq",),
+    "hqq": ("hqq",),
+    "inc": ("inc",),
+    "torchao": ("torchao",),
+}
+
+# Backends covered by the generic quantization integration (peft.utils.quantization_utils). Methods that call
+# resolve_quantization_backend don't need per-backend integration modules; they support everything the resolver
+# handles (merging may still be unavailable for the forward-only backends, which is a property of the backend, not of
+# the PEFT method).
+GENERIC_QUANT_BACKENDS: tuple[str, ...] = (
+    "aqlm",
+    "awq",
+    "bnb_4bit",
+    "bnb_8bit",
+    "eetq",
+    "gptq",
+    "hqq",
+    "inc",
+    "torchao",
+)
+
+# Extra config arguments required to instantiate a method's config for probing, on top of the generic arguments
+# (target_modules for adapter methods, task_type/num_virtual_tokens for prompt learning). If probing a method reports
+# "unknown" because its config could not be instantiated, add an entry here.
+PROBE_CONFIG_OVERRIDES: dict[PeftType, dict[str, Any]] = {
+    # total_step is a required argument
+    PeftType.ADALORA: {"total_step": 10},
+    # IA3 needs feedforward_modules
+    PeftType.IA3: {"feedforward_modules": []},
+    # the default block_size of 256 does not divide HIDDEN_DIM
+    PeftType.C3A: {"block_size": 16},
+    # token_indices defaults to an empty list, which trains nothing
+    PeftType.TRAINABLE_TOKENS: {"token_indices": [0, 1]},
+    # the default vector_length of 256 does not divide HIDDEN_DIM
+    PeftType.VBLORA: {"num_vectors": 32, "vector_length": 16},
+    PeftType.ADAPTION_PROMPT: {"adapter_layers": 1, "adapter_len": 4, "task_type": "CAUSAL_LM"},
+}
+
+# Methods that cannot be probed on a self-contained tiny model. Probe-based checks report "unknown" for these.
+PROBE_SKIP: dict[PeftType, str] = {
+    PeftType.XLORA: "requires pre-trained LoRA adapter checkpoints to instantiate",
+}
+
+# Method-specific config switches that are worth surfacing as "extras". This is a curated list: reporting every config
+# field would drown the relevant information in noise. Note that target_parameters is not listed here, as it is
+# already covered by the target_layer_types check.
+NOTABLE_CONFIG_FIELDS: tuple[str, ...] = (
+    "alpha_pattern",
+    "layer_replication",
+    "rank_pattern",
+    "use_dora",
+    "use_rslora",
+)
+
+# Docs page slugs that differ from the lower-cased PEFT method name.
+DOCS_SLUG_OVERRIDES: dict[str, str] = {
+    "ADAPTION_PROMPT": "llama_adapter",
+    "CARTRIDGE": "cartridges",
+    "LN_TUNING": "layernorm_tuning",
+}
+
+# paper links as they appear in the docs intro paragraphs and in the config/model class docstrings
+PAPER_URL_RE = re.compile(
+    r"https://(?:huggingface\.co/papers/|arxiv\.org/(?:abs|pdf)/|openreview\.net/forum\?id=)[^\s)\"'>]+"
+)
+
+
+class Source(enum.StrEnum):
+    INTROSPECTION = "introspection"
+    FILE_CHECK = "file_check"
+    PROBE = "probe"
+    ERROR = "error"
+
+
+@dataclass(frozen=True)
+class Finding:
+    value: Any
+    source: Source
+    note: str | None = None
+
+    def to_json(self) -> dict[str, Any]:
+        result: dict[str, Any] = {"value": self.value, "source": str(self.source)}
+        if self.note:
+            result["note"] = self.note
+        return result
+
+
+@dataclass(frozen=True)
+class MethodInfo:
+    peft_type: PeftType
+    config_cls: type[PeftConfig]
+    model_cls: type | None
+    category: str  # "adapter", "prompt_learning", or "other"
+
+    @property
+    def name(self) -> str:
+        return self.peft_type.value
+
+    @classmethod
+    def from_peft_type(cls, peft_type: PeftType) -> "MethodInfo":
+        config_cls = PEFT_TYPE_TO_CONFIG_MAPPING[peft_type]
+        model_cls = PEFT_TYPE_TO_TUNER_MAPPING.get(peft_type)
+        if issubclass(config_cls, PromptLearningConfig):
+            category = "prompt_learning"
+        elif (model_cls is not None) and issubclass(model_cls, BaseTuner):
+            category = "adapter"
+        else:
+            # e.g. adaption prompt, whose model class manages adapters without subclassing BaseTuner
+            category = "other"
+        return cls(peft_type=peft_type, config_cls=config_cls, model_cls=model_cls, category=category)
+
+
+def _layer_classes(method: MethodInfo) -> list[type[BaseTunerLayer]]:
+    """Return the tuner layer classes defined in the method's main layer module.
+
+    Quantization-specific layer variants (bnb.py etc.) are deliberately not considered: importing them depends on the
+    installed quantization libraries, and the main layer module is what determines baseline support.
+    """
+    tuner_layer_cls = getattr(method.model_cls, "tuner_layer_cls", None)
+    if tuner_layer_cls is None:
+        return []
+    module = sys.modules[tuner_layer_cls.__module__]
+    return [
+        obj
+        for obj in vars(module).values()
+        if isinstance(obj, type) and issubclass(obj, BaseTunerLayer) and obj.__module__ == module.__name__
+    ]
+
+
+def _format_exception(exc: BaseException, limit: int = 250) -> str:
+    msg = f"{type(exc).__name__}: {exc}"
+    return msg if len(msg) <= limit else msg[: limit - 3] + "..."
+
+
+class ProbeError(Exception):
+    """Raised when a probe cannot be set up; results in an 'unknown' finding, never in a false positive/negative."""
+
+
+class SingleLayerModel(nn.Module):
+    """Minimal host model providing a single named module ("layer") for PEFT to target."""
+
+    def __init__(self, layer: nn.Module) -> None:
+        super().__init__()
+        self.layer = layer
+
+    def forward(self, x: torch.Tensor) -> torch.Tensor:
+        return self.layer(x)
+
+
+@dataclass(frozen=True)
+class LayerSpec:
+    label: str
+    build: Callable[[], nn.Module]
+
+
+# The layer types whose support is probed per method. Probing checks injection only (i.e. whether the layer gets
+# wrapped), no forward pass, since a successful wrap is the support signal and a broken forward would be a bug.
+LAYER_SPECS: tuple[LayerSpec, ...] = (
+    LayerSpec("Linear", lambda: nn.Linear(HIDDEN_DIM, HIDDEN_DIM)),
+    LayerSpec("Embedding", lambda: nn.Embedding(16, HIDDEN_DIM)),
+    LayerSpec("Conv1d", lambda: nn.Conv1d(HIDDEN_DIM, HIDDEN_DIM, 3)),
+    LayerSpec("Conv2d", lambda: nn.Conv2d(HIDDEN_DIM, HIDDEN_DIM, 3)),
+    LayerSpec("Conv3d", lambda: nn.Conv3d(HIDDEN_DIM, HIDDEN_DIM, 3)),
+    LayerSpec("LayerNorm", lambda: nn.LayerNorm(HIDDEN_DIM)),
+    LayerSpec("MultiheadAttention", lambda: nn.MultiheadAttention(HIDDEN_DIM, num_heads=4)),
+    LayerSpec("Conv1D (transformers)", lambda: Conv1D(HIDDEN_DIM, HIDDEN_DIM)),
+)
+
+
+class ProbeContext:
+    """Builds tiny throwaway models to exercise features on CPU.
+
+    The tiny transformer used for prompt learning methods is constructed from a config (no download) and cached; each
+    probe receives a deepcopy so that probes cannot contaminate each other.
+    """
+
+    def __init__(self) -> None:
+        self._tiny_lm: nn.Module | None = None
+
+    def make_config(self, method: MethodInfo, **kwargs: Any) -> PeftConfig:
+        if method.peft_type in PROBE_SKIP:
+            raise ProbeError(f"not probed: {PROBE_SKIP[method.peft_type]}")
+        kwargs = PROBE_CONFIG_OVERRIDES.get(method.peft_type, {}) | kwargs
+        try:
+            return method.config_cls(**kwargs)
+        except Exception as exc:
+            raise ProbeError(
+                f"could not instantiate {method.config_cls.__name__} for probing "
+                f"(consider adding an entry to PROBE_CONFIG_OVERRIDES): {_format_exception(exc)}"
+            ) from exc
+
+    def _probe_layer_and_input(self, method: MethodInfo) -> tuple[nn.Module, torch.Tensor]:
+        if method.peft_type == PeftType.TRAINABLE_TOKENS:
+            # trainable tokens only target embedding layers
+            return nn.Embedding(16, HIDDEN_DIM), torch.randint(0, 16, (2, 5))
+        return nn.Linear(HIDDEN_DIM, HIDDEN_DIM), torch.randn(2, HIDDEN_DIM)
+
+    def adapter_model(self, method: MethodInfo) -> tuple[nn.Module, torch.Tensor]:
+        """Return a PEFT model wrapping a single-layer host, plus a suitable example input."""
+        torch.manual_seed(0)
+        layer, example_input = self._probe_layer_and_input(method)
+        host = SingleLayerModel(layer)
+        config = self.make_config(method, target_modules=["layer"])
+        try:
+            return get_peft_model(host, config), example_input
+        except Exception as exc:
+            raise ProbeError(f"could not build probe model: {_format_exception(exc)}") from exc
+
+    def transformer_model(self, method: MethodInfo) -> nn.Module:
+        """Return a PEFT model on a tiny transformer, for methods that require one (prompt learning etc.)."""
+        if self._tiny_lm is None:
+            torch.manual_seed(0)
+            tiny_config = LlamaConfig(
+                vocab_size=64,
+                hidden_size=32,
+                intermediate_size=64,
+                num_hidden_layers=2,
+                num_attention_heads=4,
+                num_key_value_heads=4,
+                max_position_embeddings=64,
+            )
+            self._tiny_lm = LlamaForCausalLM(tiny_config)
+        kwargs: dict[str, Any] = {"task_type": "CAUSAL_LM"}
+        if method.category == "prompt_learning":
+            kwargs["num_virtual_tokens"] = 4
+        config = self.make_config(method, **kwargs)
+        try:
+            return get_peft_model(deepcopy(self._tiny_lm), config)
+        except Exception as exc:
+            raise ProbeError(f"could not build probe model: {_format_exception(exc)}") from exc
+
+    def second_config(self, method: MethodInfo) -> PeftConfig:
+        """A config suitable for adding a second adapter to a model built by this context."""
+        if method.category == "adapter":
+            return self.make_config(method, target_modules=["layer"])
+        kwargs: dict[str, Any] = {"task_type": "CAUSAL_LM"}
+        if method.category == "prompt_learning":
+            kwargs["num_virtual_tokens"] = 4
+        return self.make_config(method, **kwargs)
+
+
+class Task(ABC):
+    """A single feature check for a single method. Never raises; failures become 'unknown' findings."""
+
+    feature: ClassVar[str]
+    description: ClassVar[str]
+
+    def __init__(self, method: MethodInfo, probe: ProbeContext) -> None:
+        self.method = method
+        self.probe = probe
+
+    @abstractmethod
+    def check(self) -> Finding: ...
+
+    def run(self) -> Finding:
+        try:
+            # Probing emits plenty of warnings that are expected and irrelevant here (e.g. about adapter
+            # initialization or fan_in_fan_out). Suppression is re-asserted per task instead of once globally, since
+            # libraries imported lazily during probing may manipulate the global warning filters.
+            with warnings.catch_warnings():
+                warnings.simplefilter("ignore")
+                return self.check()
+        except ProbeError as exc:
+            return Finding(value=UNKNOWN, source=Source.PROBE, note=str(exc))
+        except Exception as exc:
+            return Finding(value=UNKNOWN, source=Source.ERROR, note=_format_exception(exc))
+
+
+class CategoryTask(Task):
+    feature = "category"
+    description = "whether the method is a layer-wrapping adapter, a prompt learning method, or something else"
+
+    def check(self) -> Finding:
+        return Finding(value=self.method.category, source=Source.INTROSPECTION)
+
+
+class TargetLayerTypesTask(Task):
+    feature = "target_layer_types"
+    description = (
+        "which layer types (incl. nn.Parameter) can be targeted, probed by injecting into single-layer models"
+    )
+
+    def check(self) -> Finding:
+        if self.method.category != "adapter":
+            return Finding(
+                value=NOT_APPLICABLE, source=Source.INTROSPECTION, note="method does not wrap target layers"
+            )
+
+        results: dict[str, bool] = {}
+        first_error: str | None = None
+        for spec in LAYER_SPECS:
+            torch.manual_seed(0)
+            host = SingleLayerModel(spec.build())
+            config = self.probe.make_config(self.method, target_modules=["layer"])
+            try:
+                model = get_peft_model(host, config)
+            except Exception as exc:
+                results[spec.label] = False
+                if first_error is None:
+                    first_error = f"{spec.label}: {_format_exception(exc)}"
+            else:
+                results[spec.label] = any(isinstance(module, BaseTunerLayer) for module in model.modules())
+
+        if not any(results.values()):
+            # if not even one layer type can be wrapped, the probe setup is likely at fault, not the method
+            raise ProbeError(f"no layer type could be wrapped, probe presumably mis-configured; {first_error}")
+
+        # Directly targeting nn.Parameter (crucial e.g. for MoE layers) is governed by the target_parameters config
+        # option; its presence is the support signal, no injection probe is needed.
+        field_names = {f.name for f in dataclasses.fields(self.method.config_cls)}
+        results["nn.Parameter"] = "target_parameters" in field_names
+        return Finding(
+            value=results,
+            source=Source.PROBE,
+            note="nn.Parameter support is based on the presence of the target_parameters config option",
+        )
+
+
+class QuantizationTask(Task):
+    feature = "quantization_backends"
+    description = "supported quantization backends, from integration modules and use of the generic backend resolver"
+
+    def check(self) -> Finding:
+        if self.method.category == "prompt_learning":
+            return Finding(
+                value=NOT_APPLICABLE,
+                source=Source.INTROSPECTION,
+                note="prompt learning does not wrap target layers and generally works regardless of quantization",
+            )
+        if self.method.category != "adapter":
+            return Finding(value=UNKNOWN, source=Source.INTROSPECTION, note="no known quantization signal")
+
+        backends: set[str] = set()
+        package_dir = Path(inspect.getfile(self.method.model_cls)).parent
+        for stem, names in QUANT_FILE_BACKENDS.items():
+            if (package_dir / f"{stem}.py").exists():
+                backends.update(names)
+
+        # methods using the generic quantization integration support all backends the resolver handles
+        module_names = {self.method.model_cls.__module__}
+        if (tuner_layer_cls := getattr(self.method.model_cls, "tuner_layer_cls", None)) is not None:
+            module_names.add(tuner_layer_cls.__module__)
+        for module_name in module_names:
+            if "resolve_quantization_backend" in inspect.getsource(sys.modules[module_name]):
+                backends.update(GENERIC_QUANT_BACKENDS)
+                break
+
+        return Finding(value=sorted(backends), source=Source.FILE_CHECK)
+
+
+class MultipleAdaptersTask(Task):
+    feature = "multiple_adapters"
+    description = "whether several adapters can be loaded on the same model, probed via add_adapter"
+
+    def check(self) -> Finding:
+        if self.method.category == "adapter":
+            model, _ = self.probe.adapter_model(self.method)
+        else:
+            model = self.probe.transformer_model(self.method)
+        try:
+            model.add_adapter("second", self.probe.second_config(self.method))
+            model.set_adapter("second")
+        except Exception as exc:
+            return Finding(value=False, source=Source.PROBE, note=_format_exception(exc))
+        return Finding(value=True, source=Source.PROBE)
+
+
+class MixedAdapterBatchesTask(Task):
+    feature = "mixed_adapter_batches"
+    description = "whether one batch can mix several adapters via the adapter_names argument"
+
+    def check(self) -> Finding:
+        if self.method.category != "adapter":
+            return Finding(
+                value=False, source=Source.INTROSPECTION, note="only supported by layer-wrapping adapter methods"
+            )
+        # Support requires both halves of the mechanism: the model must install the forward hooks that distribute
+        # adapter_names, and the tuner layers must implement _mixed_batch_forward (possibly inherited, e.g. from
+        # LoRA). Checking only one of them would over-report.
+        model_supports_hooks = hasattr(self.method.model_cls, "_enable_peft_forward_hooks")
+        layer_classes = _layer_classes(self.method)
+        layer_supports = any(hasattr(cls, "_mixed_batch_forward") for cls in layer_classes)
+        return Finding(value=model_supports_hooks and layer_supports, source=Source.INTROSPECTION)
+
+
+class MergeTask(Task):
+    feature = "merging"
+    description = "whether adapters can be merged into the base weights, verified via merge_and_unload"
+
+    def check(self) -> Finding:
+        if self.method.category == "prompt_learning":
+            return Finding(
+                value=False, source=Source.INTROSPECTION, note="virtual tokens cannot be merged into base weights"
+            )
+        if self.method.category != "adapter":
+            return Finding(value=False, source=Source.INTROSPECTION, note="method does not implement merging")
+
+        layer_classes = _layer_classes(self.method)
+        # BaseTunerLayer.merge raises NotImplementedError, so an unchanged merge attribute means no support
+        implemented = any(cls.merge is not BaseTunerLayer.merge for cls in layer_classes)
+        if not implemented:
+            return Finding(value=False, source=Source.INTROSPECTION, note="no tuner layer class implements merge()")
+
+        try:
+            model, example_input = self.probe.adapter_model(self.method)
+            model.eval()
+            with torch.no_grad():
+                output_before = model(example_input)
+                merged = model.merge_and_unload()
+                output_after = merged(example_input)
+        except ProbeError as exc:
+            return Finding(
+                value=True, source=Source.INTROSPECTION, note=f"merge() is implemented, but probing failed: {exc}"
+            )
+        except NotImplementedError as exc:
+            return Finding(value=False, source=Source.PROBE, note=_format_exception(exc))
+        except Exception as exc:
+            return Finding(
+                value=True,
+                source=Source.INTROSPECTION,
+                note=f"merge() is implemented, but probing failed: {_format_exception(exc)}",
+            )
+
+        note = None
+        if not torch.allclose(output_before, output_after, atol=1e-4):
+            note = "merged model outputs deviate from unmerged outputs beyond tolerance"
+        return Finding(value=True, source=Source.PROBE, note=note)
+
+
+class MixedMethodModelTask(Task):
+    feature = "peft_mixed_model"
+    description = "whether the method can be combined with other method types in a PeftMixedModel"
+
+    def check(self) -> Finding:
+        # filled by register_peft_method(..., is_mixed_compatible=True)
+        value = self.method.peft_type in PEFT_TYPE_TO_MIXED_MODEL_MAPPING
+        return Finding(value=value, source=Source.INTROSPECTION)
+
+
+class LoraConversionTask(Task):
+    feature = "lora_conversion"
+    description = "whether adapters of this method can be converted to a LoRA adapter"
+
+    def check(self) -> Finding:
+        if self.method.peft_type == PeftType.LORA:
+            return Finding(value=True, source=Source.INTROSPECTION, note="already a LoRA adapter")
+        if self.method.category != "adapter":
+            return Finding(value=False, source=Source.INTROSPECTION, note="method does not wrap target layers")
+
+        try:
+            model, _ = self.probe.adapter_model(self.method)
+        except ProbeError as exc:
+            # fall back to a static signal; an override does not guarantee support, hence the note
+            layer_classes = _layer_classes(self.method)
+            overridden = any(
+                cls.supports_lora_conversion is not BaseTunerLayer.supports_lora_conversion for cls in layer_classes
+            )
+            return Finding(
+                value=overridden,
+                source=Source.INTROSPECTION,
+                note=f"based on presence of a supports_lora_conversion override; probing failed: {exc}",
+            )
+
+        layer = next(module for module in model.modules() if isinstance(module, BaseTunerLayer))
+        return Finding(value=bool(layer.supports_lora_conversion("default")), source=Source.PROBE)
+
+
+class WeightedAdapterTask(Task):
+    feature = "add_weighted_adapter"
+    description = "whether several adapters can be combined into a new one, probed via add_weighted_adapter"
+
+    def check(self) -> Finding:
+        # Methods without the API don't support the feature; methods that inherit it but override it with a stub
+        # that raises (e.g. AdaLoRA) are caught by the probe below.
+        if getattr(self.method.model_cls, "add_weighted_adapter", None) is None:
+            return Finding(value=False, source=Source.INTROSPECTION)
+
+        model, _ = self.probe.adapter_model(self.method)
+        # combining several adapters requires loading several adapters in the first place; methods that already fail
+        # here cannot support add_weighted_adapter either
+        try:
+            model.add_adapter("second", self.probe.second_config(self.method))
+        except Exception as exc:
+            return Finding(
+                value=False,
+                source=Source.PROBE,
+                note=f"a second adapter could not be added: {_format_exception(exc)}",
+            )
+        try:
+            model.add_weighted_adapter(adapters=["default", "second"], weights=[0.5, 0.5], adapter_name="combined")
+        except Exception as exc:
+            return Finding(value=False, source=Source.PROBE, note=_format_exception(exc))
+        return Finding(value=True, source=Source.PROBE)
+
+
+class HotswapTask(Task):
+    feature = "hotswapping"
+    description = "whether adapters can be hot-swapped in place (peft.utils.hotswap)"
+
+    def check(self) -> Finding:
+        return Finding(value=self.method.peft_type in CONFIG_KEYS_TO_CHECK, source=Source.INTROSPECTION)
+
+
+class AuxiliaryModulesTask(Task):
+    feature = "auxiliary_modules"
+    description = "whether the config supports modules_to_save and trainable_token_indices"
+
+    def check(self) -> Finding:
+        field_names = {f.name for f in dataclasses.fields(self.method.config_cls)}
+        value = {
+            "modules_to_save": "modules_to_save" in field_names,
+            "trainable_token_indices": "trainable_token_indices" in field_names,
+        }
+        return Finding(value=value, source=Source.INTROSPECTION)
+
+
+class ExtrasTask(Task):
+    feature = "extras"
+    description = "notable method-specific config options (curated list)"
+
+    def check(self) -> Finding:
+        field_names = {f.name for f in dataclasses.fields(self.method.config_cls)}
+        value = sorted(field_names.intersection(NOTABLE_CONFIG_FIELDS))
+        return Finding(value=value, source=Source.INTROSPECTION)
+
+
+class PaperLinkTask(Task):
+    feature = "paper_url"
+    description = "link to the method's paper, from the docs intro or class docstrings (omitted when ambiguous)"
+
+    # default assumes the script lives in scripts/ of a repository checkout; can be overridden via --docs-dir
+    docs_dir: ClassVar[Path] = Path(__file__).parent.parent / "docs" / "source" / "package_reference"
+
+    def _docs_intro(self) -> str | None:
+        """The first paragraph after the first heading of the method's docs page (the pages start with a license
+        comment, a `# Title` heading, and a prose paragraph)."""
+        slug = DOCS_SLUG_OVERRIDES.get(self.method.name, self.method.name.lower())
+        try:
+            text = (self.docs_dir / f"{slug}.md").read_text()
+        except OSError:
+            return None
+        text = re.sub(r"<!--.*?-->", "", text, flags=re.DOTALL)
+        match = re.search(r"^# .+?$\s+(.+?)(?:\n\s*\n|$)", text, flags=re.MULTILINE | re.DOTALL)
+        return match.group(1) if match else None
+
+    def check(self) -> Finding:
+        # Sources in order of preference; the first one containing exactly one distinct paper URL wins. A source
+        # with no link, or with several different ones (e.g. docstrings that also cite related methods), is skipped
+        # as ambiguous -- a missing paper link is better than a wrong one.
+        sources: list[tuple[str, str | None]] = [
+            ("docs intro", self._docs_intro()),
+            ("config class docstring", self.method.config_cls.__doc__),
+            ("model class docstring", self.method.model_cls.__doc__ if self.method.model_cls is not None else None),
+        ]
+        for source_name, text in sources:
+            if not text:
+                continue
+            urls = set(PAPER_URL_RE.findall(text))
+            if len(urls) == 1:
+                return Finding(value=urls.pop(), source=Source.FILE_CHECK, note=f"from the {source_name}")
+        return Finding(
+            value=None,
+            source=Source.FILE_CHECK,
+            note="no unambiguous paper link in the docs intro or the config/model class docstrings",
+        )
+
+
+# the order here determines the order of the features in the output
+TASK_CLASSES: tuple[type[Task], ...] = (
+    CategoryTask,
+    TargetLayerTypesTask,
+    QuantizationTask,
+    MultipleAdaptersTask,
+    MixedAdapterBatchesTask,
+    MergeTask,
+    MixedMethodModelTask,
+    LoraConversionTask,
+    WeightedAdapterTask,
+    HotswapTask,
+    AuxiliaryModulesTask,
+    ExtrasTask,
+    PaperLinkTask,
+)
+
+
+def collect_methods(selected: list[str] | None) -> list[MethodInfo]:
+    registered = sorted(PEFT_TYPE_TO_CONFIG_MAPPING, key=lambda peft_type: peft_type.value)
+    if selected is not None:
+        valid = {peft_type.value for peft_type in registered}
+        requested = [name.upper() for name in selected]
+        if unknown := [name for name in requested if name not in valid]:
+            raise SystemExit(
+                f"Unknown PEFT method(s): {', '.join(unknown)}. Valid choices: {', '.join(sorted(valid))}"
+            )
+        registered = [peft_type for peft_type in registered if peft_type.value in requested]
+    return [MethodInfo.from_peft_type(peft_type) for peft_type in registered]
+
+
+def collect_tasks(methods: list[MethodInfo], probe: ProbeContext) -> list[Task]:
+    return [task_cls(method, probe) for method in methods for task_cls in TASK_CLASSES]
+
+
+def run_tasks(tasks: list[Task]) -> dict[str, dict[str, Any]]:
+    results: dict[str, dict[str, Any]] = {}
+    for task in tqdm(tasks, desc="Checking capabilities", unit="check"):
+        method = task.method
+        entry = results.setdefault(
+            method.name,
+            {
+                "config_class": method.config_cls.__name__,
+                "model_class": method.model_cls.__name__ if method.model_cls is not None else None,
+                "features": {},
+            },
+        )
+        entry["features"][task.feature] = task.run().to_json()
+    return results
+
+
+def main(argv: list[str] | None = None) -> None:
+    parser = argparse.ArgumentParser(
+        description="Generate a JSON capability matrix of all PEFT methods.",
+        epilog="Example: python scripts/generate_method_capabilities.py --methods lora ia3 --output capabilities.json",
+    )
+    parser.add_argument(
+        "--methods",
+        "-m",
+        nargs="+",
+        default=None,
+        metavar="METHOD",
+        help="restrict the analysis to these PEFT methods (case-insensitive, e.g. 'lora ia3'); default: all",
+    )
+    parser.add_argument(
+        "--output",
+        "-o",
+        type=Path,
+        default=Path("method_capabilities.json"),
+        help="output JSON file (default: %(default)s)",
+    )
+    parser.add_argument("--dry-run", action="store_true", help="only list the checks that would run")
+    parser.add_argument(
+        "--docs-dir",
+        type=Path,
+        default=PaperLinkTask.docs_dir,
+        help="directory containing the package_reference docs pages, used for the paper link check",
+    )
+    args = parser.parse_args(argv)
+    PaperLinkTask.docs_dir = args.docs_dir
+    logging.basicConfig(level=logging.INFO, format="%(message)s")  # logs to stderr
+
+    methods = collect_methods(args.methods)
+    probe = ProbeContext()
+    tasks = collect_tasks(methods, probe)
+
+    if args.dry_run:
+        for method in methods:
+            print(f"{method.name}: {', '.join(task_cls.feature for task_cls in TASK_CLASSES)}")
+        print(f"\n{len(methods)} methods x {len(TASK_CLASSES)} checks = {len(tasks)} tasks")
+        return
+
+    output = {
+        "schema_version": 1,
+        "peft_version": peft.__version__,
+        "methods": run_tasks(tasks),
+    }
+    # The results deliberately go to a file, not to stdout: probing can trigger subprocesses whose output is written
+    # directly to stdout (e.g. BOFT compiling its CUDA extension via ninja), which would interleave with the JSON.
+    args.output.write_text(json.dumps(output, indent=2) + "\n")
+    logger.info(f"Wrote capabilities of {len(methods)} methods to {args.output}")
+
+
+if __name__ == "__main__":
+    main()

scripts/launch_notebook_mp.py

@@ -0,0 +1,47 @@
+# Copyright 2023-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+# This is a minimal example of launching PEFT with Accelerate. This used to cause issues because PEFT would eagerly
+# import bitsandbytes, which initializes CUDA, resulting in:
+# > RuntimeError: Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing, you must use the
+# > 'spawn' start method
+# This script exists to ensure that this issue does not reoccur.
+
+import torch
+from accelerate import notebook_launcher
+
+import peft
+from peft.utils import infer_device
+
+
+def init():
+    class MyModule(torch.nn.Module):
+        def __init__(self):
+            super().__init__()
+            self.linear = torch.nn.Linear(1, 2)
+
+        def forward(self, x):
+            return self.linear(x)
+
+    device = infer_device()
+    model = MyModule().to(device)
+    peft.get_peft_model(model, peft.LoraConfig(target_modules=["linear"]))
+
+
+def main():
+    notebook_launcher(init, (), num_processes=2)
+
+
+if __name__ == "__main__":
+    main()

scripts/log_reports.py

@@ -0,0 +1,144 @@
+import argparse
+import json
+import os
+from datetime import UTC, datetime
+from pathlib import Path
+
+from tabulate import tabulate
+
+
+MAX_LEN_MESSAGE = 2900  # slack endpoint has a limit of 3001 characters
+
+parser = argparse.ArgumentParser()
+parser.add_argument(
+    "--slack_channel_name",
+    default="peft-ci-daily",
+)
+
+
+def main(slack_channel_name=None):
+    failed = []
+    passed = []
+
+    group_info = []
+
+    total_num_failed = 0
+    empty_file = False or len(list(Path().glob("*.log"))) == 0
+
+    total_empty_files = []
+
+    for log in Path().glob("*.log"):
+        section_num_failed = 0
+        i = 0
+        with open(log) as f:
+            for line in f:
+                line = json.loads(line)
+                i += 1
+                if line.get("nodeid", "") != "":
+                    test = line["nodeid"]
+                    if line.get("duration", None) is not None:
+                        duration = f"{line['duration']:.4f}"
+                        if line.get("outcome", "") == "failed":
+                            section_num_failed += 1
+                            failed.append([test, duration, log.name.split("_")[0]])
+                            total_num_failed += 1
+                        else:
+                            passed.append([test, duration, log.name.split("_")[0]])
+            empty_file = i == 0
+        group_info.append([str(log), section_num_failed, failed])
+        total_empty_files.append(empty_file)
+        os.remove(log)
+        failed = []
+    text = (
+        "🌞 There were no failures!"
+        if not any(total_empty_files)
+        else "Something went wrong there is at least one empty file - please check GH action results."
+    )
+    no_error_payload = {
+        "type": "section",
+        "text": {
+            "type": "plain_text",
+            "text": text,
+            "emoji": True,
+        },
+    }
+
+    message = ""
+    payload = [
+        {
+            "type": "header",
+            "text": {
+                "type": "plain_text",
+                "text": "🤗 Results of the {} PEFT scheduled tests.".format(os.environ.get("TEST_TYPE", "")),
+            },
+        },
+    ]
+    if total_num_failed > 0:
+        for i, (name, num_failed, failed_tests) in enumerate(group_info):
+            if num_failed > 0:
+                if num_failed == 1:
+                    message += f"*{name}: {num_failed} failed test*\n"
+                else:
+                    message += f"*{name}: {num_failed} failed tests*\n"
+                failed_table = []
+                for test in failed_tests:
+                    failed_table.append(test[0].split("::"))
+                failed_table = tabulate(
+                    failed_table,
+                    headers=["Test Location", "Test Case", "Test Name"],
+                    showindex="always",
+                    tablefmt="grid",
+                    maxcolwidths=[12, 12, 12],
+                )
+                message += "\n```\n" + failed_table + "\n```"
+
+            if total_empty_files[i]:
+                message += f"\n*{name}: Warning! Empty file - please check the GitHub action job *\n"
+        print(f"### {message}")
+    else:
+        payload.append(no_error_payload)
+
+    if os.environ.get("TEST_TYPE", "") != "":
+        from slack_sdk import WebClient
+
+        if len(message) > MAX_LEN_MESSAGE:
+            print(f"Truncating long message from {len(message)} to {MAX_LEN_MESSAGE}")
+            message = message[:MAX_LEN_MESSAGE] + "..."
+
+        if len(message) != 0:
+            md_report = {
+                "type": "section",
+                "text": {"type": "mrkdwn", "text": message},
+            }
+            payload.append(md_report)
+            action_button = {
+                "type": "section",
+                "text": {"type": "mrkdwn", "text": "*For more details:*"},
+                "accessory": {
+                    "type": "button",
+                    "text": {"type": "plain_text", "text": "Check Action results", "emoji": True},
+                    "url": f"https://github.com/huggingface/peft/actions/runs/{os.environ['GITHUB_RUN_ID']}",
+                },
+            }
+            payload.append(action_button)
+
+        date_report = {
+            "type": "context",
+            "elements": [
+                {
+                    "type": "plain_text",
+                    "text": f"Nightly {os.environ.get('TEST_TYPE')} test results for {datetime.now(UTC).date()}",
+                },
+            ],
+        }
+        payload.append(date_report)
+
+        print(payload)
+
+        client = WebClient(token=os.environ.get("SLACK_API_TOKEN"))
+        client.chat_postMessage(channel=f"#{slack_channel_name}", text=message, blocks=payload)
+
+
+if __name__ == "__main__":
+    args = parser.parse_args()
+    main(args.slack_channel_name)

scripts/stale.py

@@ -0,0 +1,65 @@
+# Copyright 2023 The HuggingFace Team, the AllenNLP library authors. All rights reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+"""
+Script to close stale issue. Taken in part from the AllenNLP repository.
+https://github.com/allenai/allennlp.
+"""
+
+import os
+from datetime import datetime as dt
+from datetime import timezone
+
+from github import Github
+
+
+LABELS_TO_EXEMPT = [
+    "good first issue",
+    "good second issue",
+    "good difficult issue",
+    "feature request",
+    "new model",
+    "wip",
+    "PRs welcome to address this",
+]
+
+
+def main():
+    g = Github(os.environ["GITHUB_TOKEN"])
+    repo = g.get_repo("huggingface/peft")
+    open_issues = repo.get_issues(state="open")
+
+    for issue in open_issues:
+        comments = sorted(issue.get_comments(), key=lambda i: i.created_at, reverse=True)
+        last_comment = comments[0] if len(comments) > 0 else None
+        if (
+            (last_comment is not None and last_comment.user.login == "github-actions[bot]")
+            and (dt.now(timezone.utc) - issue.updated_at).days > 7
+            and (dt.now(timezone.utc) - issue.created_at).days >= 30
+            and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels())
+        ):
+            issue.edit(state="closed")
+        elif (
+            (dt.now(timezone.utc) - issue.updated_at).days > 23
+            and (dt.now(timezone.utc) - issue.created_at).days >= 30
+            and not any(label.name.lower() in LABELS_TO_EXEMPT for label in issue.get_labels())
+        ):
+            issue.create_comment(
+                "This issue has been automatically marked as stale because it has not had "
+                "recent activity. If you think this still needs to be addressed "
+                "please comment on this thread.\n\n"
+            )
+
+
+if __name__ == "__main__":
+    main()

scripts/train_memory.py

@@ -0,0 +1,276 @@
+# Copyright 2025-present the HuggingFace Inc. team.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+"""This script trains a model on a small text dataset and measures the memory consumption, as well as a few other
+useful metrics.
+
+Example:
+
+Get help:
+
+```bash
+python train_memory.py --help
+```
+
+Train the google/gemma-2-2b model with a LoRA config json at the indicated location.
+
+```bash
+python train_memory.py "google/gemma-2-2b" --max_seq_length 256 --batch_size 1 --rank 32 --dtype bfloat16 --path_config <path-to-adapter-config.json>
+```
+
+Fully fine-tune the model (i.e. without LoRA) by setting the rank to 0:
+
+```bash
+python train_memory.py "google/gemma-2-2b" --rank 0
+```
+
+Get an estimate of the size of the hidden states by passing `--monitor_tensors`. This trains just for a single epoch. For realistic estimates, the batch size for this:
+
+```bash
+python train_memory.py "google/gemma-2-2b" --max_seq_length 256 --batch_size 32 --rank 32 --dtype bfloat16 --path_config configs/lora_rank-32_embedding-lora/ --monitor_tensors
+```
+
+"""
+
+import argparse
+import gc
+import os
+import sys
+import tempfile
+import time
+import warnings
+from collections import Counter
+from contextlib import nullcontext
+from functools import partial
+
+import torch
+from datasets import load_dataset
+from torch import nn
+from transformers import (
+    AutoModelForCausalLM,
+    AutoTokenizer,
+    BitsAndBytesConfig,
+)
+
+from peft import LoraConfig, get_peft_model, prepare_model_for_kbit_training
+from peft.utils import CONFIG_NAME, SAFETENSORS_WEIGHTS_NAME
+
+
+# suppress all warnings
+warnings.filterwarnings("ignore")
+
+device = torch.accelerator.current_accelerator().type if hasattr(torch, "accelerator") else "cuda"
+dtype_to_bytes_linear = {"float32": 4, "float16": 2, "bfloat16": 2, "int8": 1, "int4": 0.5}
+
+
+def init_accelerator():
+    torch.manual_seed(0)
+    if device == "cpu":
+        return
+
+    device_module = getattr(torch, device, torch.cuda)
+    device_module.reset_peak_memory_stats()
+    device_module.manual_seed_all(0)
+    # might not be necessary, but just to be sure
+    nn.Linear(1, 1).to(device)
+
+
+def get_data(tokenizer):
+    def tokenize(samples):
+        # For some reason, the max sequence length is not honored by the tokenizer, resulting in IndexErrors. Thus,
+        # manually ensure that sequences are not too long.
+        tokenized = tokenizer(samples["quote"])
+        tokenized["input_ids"] = [input_ids[: tokenizer.model_max_length] for input_ids in tokenized["input_ids"]]
+        tokenized["attention_mask"] = [
+            input_ids[: tokenizer.model_max_length] for input_ids in tokenized["attention_mask"]
+        ]
+        return tokenized
+
+    data = load_dataset("ybelkada/english_quotes_copy")
+    data = data.map(tokenize, batched=True)
+    # We need to manually remove unused columns. This is because we cannot use remove_unused_columns=True in the
+    # Trainer, as this leads to errors with torch.compile. We also cannot just leave them in, as they contain
+    # strings. Therefore, manually remove all unused columns.
+    data = data.remove_columns(["quote", "author", "tags"])
+    return data
+
+
+def train(model_id, rank, dtype, monitor_tensors, max_seq_length, batch_size, max_steps, path_config):
+    init_accelerator()
+    device_module = getattr(torch, device, torch.cuda)
+    accelerator_memory_init = device_module.max_memory_allocated()
+    accelerator_memory_log = []
+
+    tokenizer = AutoTokenizer.from_pretrained(model_id)
+    tokenizer.model_max_length = max_seq_length
+    if not tokenizer.pad_token:
+        tokenizer.pad_token = tokenizer.eos_token
+    data = get_data(tokenizer)
+
+    if dtype == "int4":
+        quant_config = BitsAndBytesConfig(load_in_4bit=True)
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map=device, quantization_config=quant_config)
+        model = prepare_model_for_kbit_training(model)
+    elif dtype == "int8":
+        quant_config = BitsAndBytesConfig(load_in_8bit=True)
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map=device, quantization_config=quant_config)
+        model = prepare_model_for_kbit_training(model)
+    elif dtype == "bfloat16":
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map=device, torch_dtype=torch.bfloat16)
+    elif dtype == "float16":
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map=device, torch_dtype=torch.float16)
+    elif dtype == "float32":
+        model = AutoModelForCausalLM.from_pretrained(model_id, device_map=device)
+    else:
+        raise ValueError(f"Invalid dtype: {dtype}")
+
+    if rank > 0:
+        if path_config is None:
+            raise RuntimeError("LoRA rank > 0 requires a path to a LoRA config")
+        if path_config.endswith(CONFIG_NAME):
+            path_config = path_config.removesuffix(CONFIG_NAME)
+        config = LoraConfig.from_pretrained(path_config)
+        model = get_peft_model(model, config)
+        model.print_trainable_parameters()
+    else:
+        print("Not using LoRA")
+
+    model.config.use_cache = False
+    storage = []
+
+    def pack(x):
+        storage.append(x)
+        return len(storage) - 1
+
+    def unpack(x):
+        return storage[x]
+
+    train_ctx = partial(torch.autograd.graph.saved_tensors_hooks, pack, unpack) if monitor_tensors else nullcontext
+
+    optimizer = torch.optim.AdamW(model.parameters(), lr=1e-5)
+    losses = []
+    sample = 0
+    tic_total = time.perf_counter()
+    for i in range(max_steps):
+        storage.clear()
+        tic = time.perf_counter()
+        try:
+            batch = tokenizer.pad(data["train"][sample : sample + batch_size], return_tensors="pt").to(model.device)
+            sample += batch_size
+
+            # add targets
+            batch["labels"] = batch["input_ids"].clone()
+            optimizer.zero_grad()
+
+            with train_ctx():
+                outputs = model(**batch)
+            loss = outputs.loss
+            loss.backward()
+            optimizer.step()
+            losses.append(loss.item())
+            accelerator_memory_log.append(device_module.memory_allocated() - accelerator_memory_init)
+            device_module.empty_cache()
+            gc.collect()
+            toc = time.perf_counter()
+            print(f"step {i:3d} loss {loss.item():.6f} time {toc - tic:.2f}s", file=sys.stderr)
+        except KeyboardInterrupt:
+            print("canceled training")
+            break
+
+        if monitor_tensors:
+            break
+
+    toc_total = time.perf_counter()
+
+    accelerator_memory_final = device_module.max_memory_allocated()
+    accelerator_memory_avg = int(sum(accelerator_memory_log) / len(accelerator_memory_log))
+    print(f"{model.device.type} memory avg: {accelerator_memory_avg // 2**20}MB")
+    print(f"{model.device.type} memory max: {(accelerator_memory_final - accelerator_memory_init) // 2**20}MB")
+    print(f"total time: {toc_total - tic_total:.2f}s")
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        model.save_pretrained(tmp_dir)
+        stat = os.stat(os.path.join(tmp_dir, SAFETENSORS_WEIGHTS_NAME))
+    file_size = stat.st_size
+    print(f"file size: {file_size / 2**20:.1f}MB")
+
+    if monitor_tensors:
+        dtype_counts = Counter(t.dtype for t in storage)
+        shape_counts = Counter(t.shape for t in storage)
+        param_shape_counts = Counter(p.shape for p in model.parameters())
+        param_shape_counts_copy = dict(param_shape_counts).copy()
+
+        # shape counts includes the params, so we need to subtract them; note that they can be transposed
+        # this is an approximation
+        diff_shape_counts = {}
+        for shape, count in shape_counts.items():
+            if shape in param_shape_counts_copy:
+                diff_count = count - param_shape_counts[shape]
+                if diff_count > 0:
+                    diff_shape_counts[shape] = diff_count
+                    param_shape_counts_copy[shape] = max(0, param_shape_counts_copy[shape] - diff_count)
+            elif shape[::-1] in param_shape_counts:
+                diff_count = count - param_shape_counts[shape[::-1]]
+                if diff_count > 0:
+                    diff_shape_counts[shape] = diff_count
+                    param_shape_counts_copy[shape[::-1]] = max(0, param_shape_counts_copy[shape[::-1]] - diff_count)
+            else:
+                diff_shape_counts[shape] = count
+
+        total_size = sum(t.numel() * t.element_size() for t in storage)
+        total_size_mb = f"{total_size // 2**20}MB"
+        diff_size = 0
+        for shape, count in diff_shape_counts.items():
+            diff_size += count * torch.zeros(shape).numel() * dtype_to_bytes_linear[dtype]
+        param_size = total_size - diff_size
+
+        diff_size_mb = f"{diff_size // 2**20}MB"
+        param_size_mb = f"{param_size // 2**20}MB"
+
+        print(f"Dtype counts: {dtype_counts.most_common()}")
+        print(f"Total size of tensors:     {total_size_mb: >12}")
+        print(f"Total size of activations: {diff_size_mb: >12}")
+        print(f"Total size of parameters:  {param_size_mb: >12}")
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser()
+    parser.add_argument("model_id", type=str, help="Model name on Hugging Face Hub")
+    parser.add_argument("--rank", type=int, default=8, help="Rank of LoRA, 0 => no LoRA, default 8")
+    parser.add_argument(
+        "--dtype",
+        type=str,
+        default="float32",
+        help="Data type, one of float32, float16, bfloat16, int8, int4, default float32",
+    )
+    parser.add_argument(
+        "--monitor_tensors",
+        action="store_true",
+        help="Monitor tensor sizes during training for a single training step, off by default",
+    )
+    parser.add_argument("--max_seq_length", type=int, default=128, help="Maximum sequence length, default 128")
+    parser.add_argument("--batch_size", type=int, default=1, help="Batch size, default 1")
+    parser.add_argument("--max_steps", type=int, default=50, help="Maximum number of training steps, default 50")
+    parser.add_argument("--path_config", type=str, default=None, help="Path to LoRA config")
+    args = parser.parse_args()
+    train(
+        model_id=args.model_id,
+        rank=args.rank,
+        dtype=args.dtype,
+        monitor_tensors=args.monitor_tensors,
+        max_seq_length=args.max_seq_length,
+        batch_size=args.batch_size,
+        max_steps=args.max_steps,
+        path_config=args.path_config,
+    )