wehub-resource-sync
f877c37fc6
tests / Test (macos-latest, 3.14) (push) Has been cancelled
tests / Test (ubuntu-latest, 3.10) (push) Has been cancelled
tests / Test (ubuntu-latest, 3.11) (push) Has been cancelled
tests / Test (windows-latest, 3.13) (push) Has been cancelled
tests / Test (windows-latest, 3.14) (push) Has been cancelled
tests / Validate (push) Has been cancelled
tests / Test (macos-latest, 3.10) (push) Has been cancelled
tests / Test (macos-latest, 3.11) (push) Has been cancelled
tests / Test (macos-latest, 3.12) (push) Has been cancelled
tests / Test (macos-latest, 3.13) (push) Has been cancelled
tests / Test (ubuntu-latest, 3.12) (push) Has been cancelled
tests / Test (ubuntu-latest, 3.13) (push) Has been cancelled
tests / Test (ubuntu-latest, 3.14) (push) Has been cancelled
tests / Test (windows-latest, 3.10) (push) Has been cancelled
tests / Test (windows-latest, 3.11) (push) Has been cancelled
tests / Test (windows-latest, 3.12) (push) Has been cancelled
universe validation / Validate (push) Has been cancelled
35 lines
1.2 KiB
Python
35 lines
1.2 KiB
Python
from typing import TYPE_CHECKING, Callable, Optional
|
|
|
|
from ..errors import Errors
|
|
from ..util import load_model, logger
|
|
|
|
if TYPE_CHECKING:
|
|
from ..language import Language
|
|
|
|
|
|
def create_copy_from_base_model(
|
|
tokenizer: Optional[str] = None,
|
|
vocab: Optional[str] = None,
|
|
) -> Callable[["Language"], "Language"]:
|
|
def copy_from_base_model(nlp):
|
|
if tokenizer:
|
|
logger.info("Copying tokenizer from: %s", tokenizer)
|
|
base_nlp = load_model(tokenizer)
|
|
if nlp.config["nlp"]["tokenizer"] == base_nlp.config["nlp"]["tokenizer"]:
|
|
nlp.tokenizer.from_bytes(base_nlp.tokenizer.to_bytes(exclude=["vocab"]))
|
|
else:
|
|
raise ValueError(
|
|
Errors.E872.format(
|
|
curr_config=nlp.config["nlp"]["tokenizer"],
|
|
base_config=base_nlp.config["nlp"]["tokenizer"],
|
|
)
|
|
)
|
|
if vocab:
|
|
logger.info("Copying vocab from: %s", vocab)
|
|
# only reload if the vocab is from a different model
|
|
if tokenizer != vocab:
|
|
base_nlp = load_model(vocab)
|
|
nlp.vocab.from_bytes(base_nlp.vocab.to_bytes())
|
|
|
|
return copy_from_base_model
|