wehub-resource-sync
48 lines
1.6 KiB
Python
48 lines
1.6 KiB
Python
# coding=utf-8
|
|
from transformers.models.layoutlm.tokenization_layoutlm import LayoutLMTokenizer
|
|
from transformers.utils import logging
|
|
|
|
|
|
logger = logging.get_logger(__name__)
|
|
|
|
VOCAB_FILES_NAMES = {"vocab_file": "vocab.txt"}
|
|
|
|
PRETRAINED_VOCAB_FILES_MAP = {
|
|
"vocab_file": {
|
|
"microsoft/layoutlmv2-base-uncased": "https://huggingface.co/microsoft/layoutlmv2-base-uncased/resolve/main/vocab.txt",
|
|
"microsoft/layoutlmv2-large-uncased": "https://huggingface.co/microsoft/layoutlmv2-large-uncased/resolve/main/vocab.txt",
|
|
}
|
|
}
|
|
|
|
|
|
PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES = {
|
|
"microsoft/layoutlmv2-base-uncased": 512,
|
|
"microsoft/layoutlmv2-large-uncased": 512,
|
|
}
|
|
|
|
|
|
PRETRAINED_INIT_CONFIGURATION = {
|
|
"microsoft/layoutlmv2-base-uncased": {"do_lower_case": True},
|
|
"microsoft/layoutlmv2-large-uncased": {"do_lower_case": True},
|
|
}
|
|
|
|
|
|
class LayoutLMv2Tokenizer(LayoutLMTokenizer):
|
|
r"""
|
|
Constructs a LayoutLMv2 tokenizer.
|
|
|
|
:class:`~transformers.LayoutLMv2Tokenizer is identical to :class:`~transformers.BertTokenizer` and runs end-to-end
|
|
tokenization: punctuation splitting + wordpiece.
|
|
|
|
Refer to superclass :class:`~transformers.BertTokenizer` for usage examples and documentation concerning
|
|
parameters.
|
|
"""
|
|
|
|
vocab_files_names = VOCAB_FILES_NAMES
|
|
pretrained_vocab_files_map = PRETRAINED_VOCAB_FILES_MAP
|
|
pretrained_init_configuration = PRETRAINED_INIT_CONFIGURATION
|
|
max_model_input_sizes = PRETRAINED_POSITIONAL_EMBEDDINGS_SIZES
|
|
|
|
def __init__(self, model_max_length=512, **kwargs):
|
|
super().__init__(model_max_length=model_max_length, **kwargs)
|