wehub-resource-sync
176 lines
7.3 KiB
Python
176 lines
7.3 KiB
Python
# Copyright (c) 2022 PaddlePaddle Authors. All Rights Reserved.
|
|
# Copyright 2019 Hugging Face inc.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import random
|
|
import tempfile
|
|
import unittest
|
|
|
|
from paddlenlp.transformers import AutoConfig
|
|
from paddlenlp.transformers.auto.configuration import CONFIG_MAPPING
|
|
from paddlenlp.transformers.bert.configuration import BertConfig
|
|
from paddlenlp.utils.env import CONFIG_NAME
|
|
|
|
from ...utils.test_module.custom_configuration import CustomConfig
|
|
|
|
|
|
class AutoConfigTest(unittest.TestCase):
|
|
def test_built_in_model_class_config(self):
|
|
config = AutoConfig.from_pretrained("bert-base-uncased")
|
|
number = random.randint(0, 10000)
|
|
self.assertEqual(config.hidden_size, 768)
|
|
|
|
config.hidden_size = number
|
|
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
config.save_pretrained(tempdir)
|
|
|
|
# there is no architectures in config.json
|
|
with open(os.path.join(tempdir, AutoConfig.config_file), "r", encoding="utf-8") as f:
|
|
config_data = json.load(f)
|
|
|
|
self.assertNotIn("architectures", config_data)
|
|
|
|
# but it can load it as the PretrainedConfig class
|
|
auto_config = AutoConfig.from_pretrained(tempdir)
|
|
self.assertEqual(auto_config.hidden_size, number)
|
|
|
|
def test_community_model_class(self):
|
|
# OPT model do not support PretrainedConfig, but can load it as the AutoConfig object
|
|
config = AutoConfig.from_pretrained("facebook/opt-125m")
|
|
|
|
self.assertEqual(config.hidden_size, 768)
|
|
|
|
number = random.randint(0, 10000)
|
|
config.hidden_size = number
|
|
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
config.save_pretrained(tempdir)
|
|
|
|
# but it can load it as the PretrainedConfig class
|
|
auto_config = AutoConfig.from_pretrained(tempdir)
|
|
self.assertEqual(auto_config.hidden_size, number)
|
|
|
|
@unittest.skip("skipping due to connection error!")
|
|
def test_from_hf_hub(self):
|
|
config = AutoConfig.from_pretrained("facebook/opt-66b", from_hf_hub=True)
|
|
self.assertEqual(config.hidden_size, 9216)
|
|
|
|
@unittest.skip("skipping due to connection error!")
|
|
def test_from_aistudio(self):
|
|
config = AutoConfig.from_pretrained("PaddleNLP/tiny-random-bert", from_aistudio=True)
|
|
self.assertEqual(config.hidden_size, 32)
|
|
|
|
# def test_subfolder(self):
|
|
# config = AutoConfig.from_pretrained("CompVis/stable-diffusion-v1-4", subfolder="text_encoder")
|
|
# self.assertEqual(config.hidden_size, 768)
|
|
|
|
def test_load_from_legacy_config(self):
|
|
number = random.randint(0, 10000)
|
|
legacy_config = {"init_class": "BertModel", "hidden_size": number}
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
with open(os.path.join(tempdir, AutoConfig.legacy_config_file), "w", encoding="utf-8") as f:
|
|
json.dump(legacy_config, f, ensure_ascii=False)
|
|
|
|
# but it can load it as the PretrainedConfig class
|
|
auto_config = AutoConfig.from_pretrained(tempdir)
|
|
self.assertEqual(auto_config.hidden_size, number)
|
|
|
|
def test_new_config_registration(self):
|
|
try:
|
|
AutoConfig.register("custom", CustomConfig)
|
|
# Wrong model type will raise an error
|
|
with self.assertRaises(ValueError):
|
|
AutoConfig.register("model", CustomConfig)
|
|
# Trying to register something existing in the PaddleNLP library will raise an error
|
|
with self.assertRaises(ValueError):
|
|
AutoConfig.register("bert", BertConfig)
|
|
|
|
# Now that the config is registered, it can be used as any other config with the auto-API
|
|
config = CustomConfig()
|
|
with tempfile.TemporaryDirectory() as tmp_dir:
|
|
config.save_pretrained(tmp_dir)
|
|
new_config = AutoConfig.from_pretrained(tmp_dir)
|
|
self.assertIsInstance(new_config, CustomConfig)
|
|
|
|
finally:
|
|
if "custom" in CONFIG_MAPPING._extra_content:
|
|
del CONFIG_MAPPING._extra_content["custom"]
|
|
|
|
def test_from_pretrained_cache_dir(self):
|
|
model_id = "__internal_testing__/tiny-random-bert"
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
AutoConfig.from_pretrained(model_id, cache_dir=tempdir)
|
|
self.assertTrue(os.path.exists(os.path.join(tempdir, model_id, CONFIG_NAME)))
|
|
# check against double appending model_name in cache_dir
|
|
self.assertFalse(os.path.exists(os.path.join(tempdir, model_id, model_id)))
|
|
|
|
def test_load_from_custom_arch(self):
|
|
config_dict = {
|
|
"alibi": False,
|
|
"architectures": ["LlamaModelForScore"],
|
|
"bias": False,
|
|
"bos_token_id": 1,
|
|
"do_normalize": False,
|
|
"eos_token_id": 2,
|
|
"fuse_attention_ffn": False,
|
|
"fuse_attention_qkv": False,
|
|
"fuse_sequence_parallel_allreduce": False,
|
|
"hidden_act": "silu",
|
|
"hidden_size": 4096,
|
|
"initializer_range": 0.02,
|
|
"intermediate_size": 11008,
|
|
"max_position_embeddings": 2048,
|
|
"model_type": "llama",
|
|
"no_recompute_layers": None,
|
|
"normalizer_type": None,
|
|
"num_attention_heads": 32,
|
|
"num_hidden_layers": 32,
|
|
"num_key_value_heads": 32,
|
|
"pad_token_id": 32000,
|
|
"paddlenlp_version": None,
|
|
"pp_recompute_interval": 1,
|
|
"recompute_granularity": "full",
|
|
"rms_norm_eps": 1e-06,
|
|
"rope_scaling_factor": 1.0,
|
|
"rope_scaling_type": None,
|
|
"score_dim": 1,
|
|
"score_type": "reward",
|
|
"seq_length": 2048,
|
|
"sequence_parallel": False,
|
|
"tensor_parallel_output": True,
|
|
"tie_word_embeddings": False,
|
|
"transformers_version": "4.28.1",
|
|
"use_flash_attention": False,
|
|
"use_fused_rms_norm": False,
|
|
"use_fused_rope": False,
|
|
"use_recompute": False,
|
|
"virtual_pp_degree": 1,
|
|
"vocab_size": 32001,
|
|
}
|
|
config_str = json.dumps(config_dict, indent=2, sort_keys=True, ensure_ascii=False) + "\n"
|
|
with tempfile.TemporaryDirectory() as tempdir:
|
|
cache_dir = os.path.join(tempdir, "cache_dir")
|
|
model_dir = os.path.join(tempdir, "custom_model")
|
|
os.mkdir(cache_dir)
|
|
os.mkdir(model_dir)
|
|
json_file_path = os.path.join(model_dir, AutoConfig.config_file)
|
|
with open(json_file_path, "w", encoding="utf-8") as writer:
|
|
writer.write(config_str)
|
|
config = AutoConfig.from_pretrained(model_dir, cache_dir=cache_dir)
|
|
self.assertTrue(config.__class__.__name__ == "LlamaConfig")
|