# Copyright 2023-present the HuggingFace Inc. team. # # Licensed under the Apache License, Version 2.0 (the "License"); # you may not use this file except in compliance with the License. # You may obtain a copy of the License at # # http://www.apache.org/licenses/LICENSE-2.0 # # Unless required by applicable law or agreed to in writing, software # distributed under the License is distributed on an "AS IS" BASIS, # WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. # See the License for the specific language governing permissions and # limitations under the License. import pytest import torch from transformers import AutoModel from peft import ( AdaLoraConfig, AdamssConfig, BeftConfig, BOFTConfig, C3AConfig, DeftConfig, DeloraConfig, FourierFTConfig, FrodConfig, GloraConfig, GraloraConfig, HiraConfig, HRAConfig, IA3Config, LilyConfig, LoraConfig, MissConfig, OFTConfig, PeanutConfig, PrefixTuningConfig, PromptEncoderConfig, PromptLearningConfig, PromptTuningConfig, PsoftConfig, RoadConfig, ShiraConfig, TinyLoraConfig, VBLoRAConfig, VeraConfig, WaveFTConfig, ) from .testing_common import PeftCommonTester from .testing_utils import set_init_weights_false # Note: models from peft-internal-testing are just the safetensors versions of hf-internal-testing PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST = [ "peft-internal-testing/tiny-random-BertModel", "peft-internal-testing/tiny-random-RobertaModel", "peft-internal-testing/tiny-random-DebertaModel", "peft-internal-testing/tiny-random-DebertaV2Model", ] # TODO Missing from this list are LoKr, LoHa, LN Tuning, add them ALL_CONFIGS = [ ( AdaLoraConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "total_step": 1, }, ), ( BeftConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( BOFTConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( MissConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "r": 2, }, ), ( DeftConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( DeloraConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "r": 2, }, ), ( FourierFTConfig, { "task_type": "FEATURE_EXTRACTION", "n_frequency": 10, "target_modules": None, }, ), ( FrodConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "sparse_rate": 0.01, }, ), ( GloraConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( GraloraConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( HiraConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( HRAConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( IA3Config, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "feedforward_modules": None, }, ), ( LilyConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "r": 8, "stride_A": 1, "num_B": 2, }, ), ( LoraConfig, { "task_type": "FEATURE_EXTRACTION", "r": 8, "lora_alpha": 32, "target_modules": None, "lora_dropout": 0.05, "bias": "none", }, ), # LoRA + trainable tokens ( LoraConfig, { "task_type": "FEATURE_EXTRACTION", "r": 8, "lora_alpha": 32, "target_modules": None, "lora_dropout": 0.05, "bias": "none", "trainable_token_indices": [0, 1, 3], }, ), ( OFTConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( PrefixTuningConfig, { "task_type": "FEATURE_EXTRACTION", "num_virtual_tokens": 10, }, ), ( PromptEncoderConfig, { "task_type": "FEATURE_EXTRACTION", "num_virtual_tokens": 10, "encoder_hidden_size": 32, }, ), ( PromptTuningConfig, { "task_type": "FEATURE_EXTRACTION", "num_virtual_tokens": 10, }, ), ( PeanutConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "r": 8, "depth": 1, "act_fn": "relu", "scaling": 1.0, }, ), ( RoadConfig, { "task_type": "FEATURE_EXTRACTION", "variant": "road_1", "group_size": 2, }, ), ( ShiraConfig, { "r": 1, "task_type": "FEATURE_EXTRACTION", "target_modules": None, "init_weights": False, }, ), ( VBLoRAConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "vblora_dropout": 0.05, "vector_length": 1, "num_vectors": 2, }, ), ( VeraConfig, { "task_type": "FEATURE_EXTRACTION", "r": 8, "target_modules": None, "vera_dropout": 0.05, "projection_prng_key": 0xFF, "d_initial": 0.1, "save_projection": True, "bias": "none", }, ), ( TinyLoraConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, }, ), ( C3AConfig, { "task_type": "FEATURE_EXTRACTION", "block_size": 1, "target_modules": None, }, ), ( WaveFTConfig, { "task_type": "FEATURE_EXTRACTION", "n_frequency": 8, "target_modules": None, }, ), ( PsoftConfig, { "task_type": "FEATURE_EXTRACTION", "r": 4, "psoft_alpha": 4, "target_modules": None, }, ), ( AdamssConfig, { "task_type": "FEATURE_EXTRACTION", "target_modules": None, "r": 8, }, ), ] def skip_non_prompt_learning(config_cls): if not issubclass(config_cls, PromptLearningConfig) or (config_cls == PrefixTuningConfig): pytest.skip("Skip tests that are not prompt learning or that are prefix tuning") def skip_deberta_lora_tests(config_cls, model_id): if "deberta" not in model_id.lower(): return to_skip = ["lora", "ia3", "boft", "vera", "fourierft", "hira", "hra", "randlora"] config_name = config_cls.__name__.lower() if any(k in config_name for k in to_skip): pytest.skip(f"Skip tests that use {config_name} for Deberta models") def deberta_beft_tests(config_cls, model_id, config_kwargs): if "deberta" not in model_id.lower(): return config_name = config_cls.__name__.lower() if config_name == "beftconfig": config_kwargs["target_modules"] = ["output.dense"] def skip_deberta_pt_tests(config_cls, model_id): if "deberta" not in model_id.lower(): return to_skip = ["prefix"] config_name = config_cls.__name__.lower() if any(k in config_name for k in to_skip): pytest.skip(f"Skip tests that use {config_name} for Deberta models") class TestPeftFeatureExtractionModel(PeftCommonTester): """ Test if the PeftModel behaves as expected. This includes: - test if the model has the expected methods """ transformers_class = AutoModel def prepare_inputs_for_testing(self): input_ids = torch.tensor([[1, 1, 1], [1, 2, 1]]).to(self.torch_device) attention_mask = torch.tensor([[1, 1, 1], [1, 0, 1]]).to(self.torch_device) input_dict = { "input_ids": input_ids, "attention_mask": attention_mask, } return input_dict @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_attributes_parametrized(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_model_attr(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_adapter_name(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_adapter_name(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_prepare_for_training_parametrized(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_prepare_for_training(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_save_pretrained(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) config_kwargs = set_init_weights_false(config_cls, config_kwargs) self._test_save_pretrained(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_save_pretrained_selected_adapters(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) config_kwargs = set_init_weights_false(config_cls, config_kwargs) self._test_save_pretrained_selected_adapters(model_id, config_cls, config_kwargs) def test_load_model_low_cpu_mem_usage(self): self._test_load_model_low_cpu_mem_usage(PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST[0], LoraConfig, {}) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_from_pretrained_config_construction(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_from_pretrained_config_construction(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_merge_layers(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) config_kwargs = set_init_weights_false(config_cls, config_kwargs) self._test_merge_layers(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_training(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_training(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_training_prompt_learning_tasks(self, model_id, config_cls, config_kwargs): skip_deberta_pt_tests(config_cls, model_id) self._test_training_prompt_learning_tasks(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_training_layer_indexing(self, model_id, config_cls, config_kwargs): self._test_training_layer_indexing(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) @pytest.mark.parametrize("use_reentrant", [True, False]) def test_training_gradient_checkpointing(self, model_id, config_cls, config_kwargs, use_reentrant): deberta_beft_tests(config_cls, model_id, config_kwargs) skip_deberta_lora_tests(config_cls, model_id) self._test_training_gradient_checkpointing(model_id, config_cls, config_kwargs, use_reentrant=use_reentrant) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_inference_safetensors(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_inference_safetensors(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_peft_model_device_map(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_peft_model_device_map(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_delete_adapter(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_delete_adapter(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_delete_inactive_adapter(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) self._test_delete_inactive_adapter(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_unload_adapter(self, model_id, config_cls, config_kwargs): deberta_beft_tests(config_cls, model_id, config_kwargs) config_kwargs = set_init_weights_false(config_cls, config_kwargs) self._test_unload_adapter(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_weighted_combination_of_adapters(self, model_id, config_cls, config_kwargs): config_kwargs = set_init_weights_false(config_cls, config_kwargs) self._test_weighted_combination_of_adapters(model_id, config_cls, config_kwargs) @pytest.mark.parametrize("model_id", PEFT_FEATURE_EXTRACTION_MODELS_TO_TEST) @pytest.mark.parametrize("config_cls,config_kwargs", ALL_CONFIGS) def test_passing_input_embeds_works(self, model_id, config_cls, config_kwargs): skip_non_prompt_learning(config_cls) self._test_passing_input_embeds_works("test input embeds work", model_id, config_cls, config_kwargs)