wehub-resource-sync
47 lines
1.6 KiB
Python
47 lines
1.6 KiB
Python
# Copyright (c) 2024 PaddlePaddle Authors. All Rights Reserved.
|
|
# Copyright 2024 The Qwen team, Alibaba Group and the HuggingFace Team. All rights reserved.
|
|
#
|
|
# Licensed under the Apache License, Version 2.0 (the "License");
|
|
# you may not use this file except in compliance with the License.
|
|
# You may obtain a copy of the License at
|
|
#
|
|
# http://www.apache.org/licenses/LICENSE-2.0
|
|
#
|
|
# Unless required by applicable law or agreed to in writing, software
|
|
# distributed under the License is distributed on an "AS IS" BASIS,
|
|
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
|
# See the License for the specific language governing permissions and
|
|
# limitations under the License.
|
|
|
|
|
|
import unittest
|
|
|
|
from paddlenlp.transformers import QWenTokenizer
|
|
|
|
|
|
class Qwen2TokenizationTest(unittest.TestCase):
|
|
from_pretrained_id = "qwen/qwen-7b"
|
|
tokenizer_class = QWenTokenizer
|
|
test_slow_tokenizer = True
|
|
space_between_special_tokens = False
|
|
from_pretrained_kwargs = None
|
|
test_seq2seq = False
|
|
|
|
def setUp(self):
|
|
super().setUp()
|
|
|
|
def get_tokenizer(self, **kwargs):
|
|
return QWenTokenizer.from_pretrained(self.from_pretrained_id, **kwargs)
|
|
|
|
def test_add_special_tokens(self):
|
|
tokenizer = self.get_tokenizer()
|
|
origin_tokens_len = len(tokenizer)
|
|
|
|
add_tokens_num = tokenizer.add_special_tokens({"additional_special_tokens": ["<img>"]})
|
|
assert add_tokens_num == 1
|
|
assert len(tokenizer) == origin_tokens_len + 1
|
|
|
|
add_tokens_num = tokenizer.add_special_tokens({"unk_token": "<unk>"})
|
|
assert add_tokens_num == 1
|
|
assert len(tokenizer) == origin_tokens_len + 2
|