chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,13 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from .configs import FairseqDataclass
|
||||
from .constants import ChoiceEnum
|
||||
|
||||
|
||||
__all__ = [
|
||||
"FairseqDataclass",
|
||||
"ChoiceEnum",
|
||||
]
|
||||
@@ -0,0 +1,925 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import sys
|
||||
from dataclasses import _MISSING_TYPE, dataclass, field
|
||||
from typing import Any, List, Optional
|
||||
|
||||
import torch
|
||||
|
||||
from fairseq.dataclass.constants import (
|
||||
DATASET_IMPL_CHOICES,
|
||||
DDP_BACKEND_CHOICES,
|
||||
GENERATION_CONSTRAINTS_CHOICES,
|
||||
GENERATION_DECODING_FORMAT_CHOICES,
|
||||
LOG_FORMAT_CHOICES,
|
||||
PIPELINE_CHECKPOINT_CHOICES,
|
||||
PRINT_ALIGNMENT_CHOICES,
|
||||
ZERO_SHARDING_CHOICES,
|
||||
)
|
||||
|
||||
from omegaconf import II, MISSING
|
||||
|
||||
|
||||
@dataclass
|
||||
class FairseqDataclass:
|
||||
"""fairseq base dataclass that supported fetching attributes and metas"""
|
||||
|
||||
_name: Optional[str] = None
|
||||
|
||||
@staticmethod
|
||||
def name():
|
||||
return None
|
||||
|
||||
def _get_all_attributes(self) -> List[str]:
|
||||
return [k for k in self.__dataclass_fields__.keys()]
|
||||
|
||||
def _get_meta(
|
||||
self, attribute_name: str, meta: str, default: Optional[Any] = None
|
||||
) -> Any:
|
||||
return self.__dataclass_fields__[attribute_name].metadata.get(meta, default)
|
||||
|
||||
def _get_name(self, attribute_name: str) -> str:
|
||||
return self.__dataclass_fields__[attribute_name].name
|
||||
|
||||
def _get_default(self, attribute_name: str) -> Any:
|
||||
if hasattr(self, attribute_name):
|
||||
if str(getattr(self, attribute_name)).startswith("${"):
|
||||
return str(getattr(self, attribute_name))
|
||||
elif str(self.__dataclass_fields__[attribute_name].default).startswith(
|
||||
"${"
|
||||
):
|
||||
return str(self.__dataclass_fields__[attribute_name].default)
|
||||
elif (
|
||||
getattr(self, attribute_name)
|
||||
!= self.__dataclass_fields__[attribute_name].default
|
||||
):
|
||||
return getattr(self, attribute_name)
|
||||
|
||||
f = self.__dataclass_fields__[attribute_name]
|
||||
if not isinstance(f.default_factory, _MISSING_TYPE):
|
||||
return f.default_factory()
|
||||
return f.default
|
||||
|
||||
def _get_type(self, attribute_name: str) -> Any:
|
||||
return self.__dataclass_fields__[attribute_name].type
|
||||
|
||||
def _get_help(self, attribute_name: str) -> Any:
|
||||
return self._get_meta(attribute_name, "help")
|
||||
|
||||
def _get_argparse_const(self, attribute_name: str) -> Any:
|
||||
return self._get_meta(attribute_name, "argparse_const")
|
||||
|
||||
def _get_argparse_alias(self, attribute_name: str) -> Any:
|
||||
return self._get_meta(attribute_name, "argparse_alias")
|
||||
|
||||
def _get_choices(self, attribute_name: str) -> Any:
|
||||
return self._get_meta(attribute_name, "choices")
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommonConfig(FairseqDataclass):
|
||||
# This is the core dataclass including common parameters shared by all different jobs. Please append your params to other dataclasses if they were
|
||||
# used for a particular purpose or task, such as those dedicated for `distributed training`, `optimization`, etc.
|
||||
no_progress_bar: bool = field(
|
||||
default=False, metadata={"help": "disable progress bar"}
|
||||
)
|
||||
log_interval: int = field(
|
||||
default=100,
|
||||
metadata={
|
||||
"help": "log progress every N batches (when progress bar is disabled)"
|
||||
},
|
||||
)
|
||||
log_format: Optional[LOG_FORMAT_CHOICES] = field(
|
||||
default=None, metadata={"help": "log format to use"}
|
||||
)
|
||||
tensorboard_logdir: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "path to save logs for tensorboard, should match --logdir "
|
||||
"of running tensorboard (default: no tensorboard logging)"
|
||||
},
|
||||
)
|
||||
wandb_project: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "Weights and Biases project name to use for logging"
|
||||
},
|
||||
)
|
||||
azureml_logging: Optional[bool] = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "Log scalars to AzureML context"
|
||||
},
|
||||
)
|
||||
seed: int = field(
|
||||
default=1, metadata={"help": "pseudo random number generator seed"}
|
||||
)
|
||||
cpu: bool = field(default=False, metadata={"help": "use CPU instead of CUDA"})
|
||||
tpu: bool = field(default=False, metadata={"help": "use TPU instead of CUDA"})
|
||||
bf16: bool = field(default=False, metadata={"help": "use bfloat16; implies --tpu"})
|
||||
memory_efficient_bf16: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "use a memory-efficient version of BF16 training; implies --bf16"
|
||||
},
|
||||
)
|
||||
fp16: bool = field(default=False, metadata={"help": "use FP16"})
|
||||
memory_efficient_fp16: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "use a memory-efficient version of FP16 training; implies --fp16"
|
||||
},
|
||||
)
|
||||
fp16_no_flatten_grads: bool = field(
|
||||
default=False, metadata={"help": "don't flatten FP16 grads tensor"}
|
||||
)
|
||||
fp16_init_scale: int = field(
|
||||
default=2 ** 7, metadata={"help": "default FP16 loss scale"}
|
||||
)
|
||||
fp16_scale_window: Optional[int] = field(
|
||||
default=None,
|
||||
metadata={"help": "number of updates before increasing loss scale"},
|
||||
)
|
||||
fp16_scale_tolerance: float = field(
|
||||
default=0.0,
|
||||
metadata={
|
||||
"help": "pct of updates that can overflow before decreasing the loss scale"
|
||||
},
|
||||
)
|
||||
min_loss_scale: float = field(
|
||||
default=1e-4,
|
||||
metadata={"help": "minimum FP16 loss scale, after which training is stopped"},
|
||||
)
|
||||
threshold_loss_scale: Optional[float] = field(
|
||||
default=None, metadata={"help": "threshold FP16 loss scale from below"}
|
||||
)
|
||||
user_dir: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "path to a python module containing custom extensions (tasks and/or architectures)"
|
||||
},
|
||||
)
|
||||
empty_cache_freq: int = field(
|
||||
default=0,
|
||||
metadata={"help": "how often to clear the PyTorch CUDA cache (0 to disable)"},
|
||||
)
|
||||
all_gather_list_size: int = field(
|
||||
default=16384,
|
||||
metadata={"help": "number of bytes reserved for gathering stats from workers"},
|
||||
)
|
||||
model_parallel_size: int = field(
|
||||
default=1, metadata={"help": "total number of GPUs to parallelize model over"}
|
||||
)
|
||||
quantization_config_path: Optional[str] = field(
|
||||
default=None, metadata={"help": "path to quantization config file"}
|
||||
)
|
||||
profile: bool = field(
|
||||
default=False, metadata={"help": "enable autograd profiler emit_nvtx"}
|
||||
)
|
||||
reset_logging: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "when using Hydra, reset the logging at the beginning of training"
|
||||
},
|
||||
)
|
||||
suppress_crashes: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "suppress crashes when training with the hydra_train entry point so that the "
|
||||
"main method can return a value (useful for sweeps)"
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DistributedTrainingConfig(FairseqDataclass):
|
||||
distributed_world_size: int = field(
|
||||
default=max(1, torch.cuda.device_count()),
|
||||
metadata={
|
||||
"help": "total number of GPUs across all nodes (default: all visible GPUs)"
|
||||
},
|
||||
)
|
||||
distributed_rank: Optional[int] = field(
|
||||
default=0, metadata={"help": "rank of the current worker"}
|
||||
)
|
||||
distributed_backend: str = field(
|
||||
default="nccl", metadata={"help": "distributed backend"}
|
||||
)
|
||||
distributed_init_method: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "typically tcp://hostname:port that will be used to "
|
||||
"establish initial connetion"
|
||||
},
|
||||
)
|
||||
distributed_port: int = field(
|
||||
default=-1,
|
||||
metadata={
|
||||
"help": "port number (not required if using --distributed-init-method)"
|
||||
},
|
||||
)
|
||||
device_id: int = field(
|
||||
default=0,
|
||||
metadata={
|
||||
"help": "which GPU to use (usually configured automatically)",
|
||||
"argparse_alias": "--local_rank",
|
||||
},
|
||||
)
|
||||
distributed_no_spawn: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "do not spawn multiple processes even if multiple GPUs are visible"
|
||||
},
|
||||
)
|
||||
ddp_backend: DDP_BACKEND_CHOICES = field(
|
||||
default="pytorch_ddp", metadata={"help": "DistributedDataParallel backend"}
|
||||
)
|
||||
bucket_cap_mb: int = field(
|
||||
default=25, metadata={"help": "bucket size for reduction"}
|
||||
)
|
||||
fix_batches_to_gpus: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "don't shuffle batches between GPUs; this reduces overall "
|
||||
"randomness and may affect precision but avoids the cost of re-reading the data"
|
||||
},
|
||||
)
|
||||
find_unused_parameters: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "disable unused parameter detection (not applicable to "
|
||||
"--ddp-backend=legacy_ddp)"
|
||||
},
|
||||
)
|
||||
fast_stat_sync: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "[deprecated] this is now defined per Criterion"},
|
||||
)
|
||||
heartbeat_timeout: int = field(
|
||||
default=-1,
|
||||
metadata={
|
||||
"help": "kill the job if no progress is made in N seconds; "
|
||||
"set to -1 to disable"
|
||||
}
|
||||
)
|
||||
broadcast_buffers: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "Copy non-trainable parameters between GPUs, such as "
|
||||
"batchnorm population statistics"
|
||||
},
|
||||
)
|
||||
slowmo_momentum: Optional[float] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "SlowMo momentum term; by default use 0.0 for 16 GPUs, "
|
||||
"0.2 for 32 GPUs; 0.5 for 64 GPUs, 0.6 for > 64 GPUs"
|
||||
},
|
||||
)
|
||||
slowmo_algorithm: str = field(
|
||||
default="LocalSGD", metadata={"help": "whether to use LocalSGD or SGP"}
|
||||
)
|
||||
localsgd_frequency: int = field(
|
||||
default=3, metadata={"help": "Local SGD allreduce frequency"}
|
||||
)
|
||||
nprocs_per_node: int = field(
|
||||
default=max(1, torch.cuda.device_count()),
|
||||
metadata={
|
||||
"help": "number of GPUs in each node. An allreduce operation across GPUs in "
|
||||
"a node is very fast. Hence, we do allreduce across GPUs in a node, "
|
||||
"and gossip across different nodes"
|
||||
},
|
||||
)
|
||||
pipeline_model_parallel: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "if set, use pipeline model parallelism across GPUs"},
|
||||
)
|
||||
pipeline_balance: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "partition the model into N_K pieces, where each piece "
|
||||
"contains N_i layers. The sum(args.pipeline_balance) "
|
||||
"should equal the total number of layers in the model"
|
||||
},
|
||||
)
|
||||
pipeline_devices: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "a list of device indices indicating which device to place "
|
||||
"each of the N_K partitions. The length of this list should "
|
||||
"equal the length of the --pipeline-balance argument"
|
||||
},
|
||||
)
|
||||
pipeline_chunks: Optional[int] = field(
|
||||
default=0, metadata={"help": "microbatch count for pipeline model parallelism"}
|
||||
)
|
||||
pipeline_encoder_balance: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "partition the pipeline parallel encoder into N_K pieces, where each piece "
|
||||
"contains N_i layers. The sum(args.pipeline_encoder_balance) "
|
||||
"should equal the total number of encoder layers in the model"
|
||||
},
|
||||
)
|
||||
pipeline_encoder_devices: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "a list of device indices indicating which device to place "
|
||||
"each of the N_K partitions. The length of this list should "
|
||||
"equal the length of the --pipeline-encoder-balance argument"
|
||||
},
|
||||
)
|
||||
pipeline_decoder_balance: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "partition the pipeline parallel decoder into N_K pieces, where each piece "
|
||||
"contains N_i layers. The sum(args.pipeline_decoder_balance) "
|
||||
"should equal the total number of decoder layers in the model"
|
||||
},
|
||||
)
|
||||
pipeline_decoder_devices: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "a list of device indices indicating which device to place "
|
||||
"each of the N_K partitions. The length of this list should "
|
||||
"equal the length of the --pipeline-decoder-balance argument"
|
||||
},
|
||||
)
|
||||
pipeline_checkpoint: PIPELINE_CHECKPOINT_CHOICES = field(
|
||||
default="never",
|
||||
metadata={"help": "checkpointing mode for pipeline model parallelism"},
|
||||
)
|
||||
zero_sharding: ZERO_SHARDING_CHOICES = field(
|
||||
default="none", metadata={"help": "ZeRO sharding"}
|
||||
)
|
||||
tpu: bool = II("common.tpu")
|
||||
|
||||
|
||||
@dataclass
|
||||
class DatasetConfig(FairseqDataclass):
|
||||
num_workers: int = field(
|
||||
default=1, metadata={"help": "how many subprocesses to use for data loading"}
|
||||
)
|
||||
skip_invalid_size_inputs_valid_test: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "ignore too long or too short lines in valid and test set"},
|
||||
)
|
||||
max_tokens: Optional[int] = field(
|
||||
default=None, metadata={"help": "maximum number of tokens in a batch"}
|
||||
)
|
||||
batch_size: Optional[int] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "number of examples in a batch",
|
||||
"argparse_alias": "--max-sentences",
|
||||
},
|
||||
)
|
||||
required_batch_size_multiple: int = field(
|
||||
default=8, metadata={"help": "batch size will be a multiplier of this value"}
|
||||
)
|
||||
required_seq_len_multiple: int = field(
|
||||
default=1,
|
||||
metadata={
|
||||
"help": "maximum sequence length in batch will be a multiplier of this value"
|
||||
},
|
||||
)
|
||||
dataset_impl: Optional[DATASET_IMPL_CHOICES] = field(
|
||||
default=None, metadata={"help": "output dataset implementation"}
|
||||
)
|
||||
data_buffer_size: int = field(
|
||||
default=10, metadata={"help": "Number of batches to preload"}
|
||||
)
|
||||
train_subset: str = field(
|
||||
default="train",
|
||||
metadata={"help": "data subset to use for training (e.g. train, valid, test)"},
|
||||
)
|
||||
valid_subset: str = field(
|
||||
default="valid",
|
||||
metadata={
|
||||
"help": "comma separated list of data subsets to use for validation"
|
||||
" (e.g. train, valid, test)"
|
||||
},
|
||||
)
|
||||
validate_interval: int = field(
|
||||
default=1, metadata={"help": "validate every N epochs"}
|
||||
)
|
||||
validate_interval_updates: int = field(
|
||||
default=0, metadata={"help": "validate every N updates"}
|
||||
)
|
||||
validate_after_updates: int = field(
|
||||
default=0, metadata={"help": "dont validate until reaching this many updates"}
|
||||
)
|
||||
fixed_validation_seed: Optional[int] = field(
|
||||
default=None, metadata={"help": "specified random seed for validation"}
|
||||
)
|
||||
disable_validation: bool = field(
|
||||
default=False, metadata={"help": "disable validation"}
|
||||
)
|
||||
max_tokens_valid: Optional[int] = field(
|
||||
default=II("dataset.max_tokens"),
|
||||
metadata={
|
||||
"help": "maximum number of tokens in a validation batch"
|
||||
" (defaults to --max-tokens)"
|
||||
},
|
||||
)
|
||||
batch_size_valid: Optional[int] = field(
|
||||
default=II("dataset.batch_size"),
|
||||
metadata={
|
||||
"help": "batch size of the validation batch (defaults to --batch-size)",
|
||||
"argparse_alias": "--max-sentences-valid",
|
||||
},
|
||||
)
|
||||
curriculum: int = field(
|
||||
default=0, metadata={"help": "don't shuffle batches for first N epochs"}
|
||||
)
|
||||
gen_subset: str = field(
|
||||
default="test",
|
||||
metadata={"help": "data subset to generate (train, valid, test)"},
|
||||
)
|
||||
num_shards: int = field(
|
||||
default=1, metadata={"help": "shard generation over N shards"}
|
||||
)
|
||||
shard_id: int = field(
|
||||
default=0, metadata={"help": "id of the shard to generate (id < num_shards)"}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class OptimizationConfig(FairseqDataclass):
|
||||
max_epoch: int = field(
|
||||
default=0, metadata={"help": "force stop training at specified epoch"}
|
||||
)
|
||||
max_update: int = field(
|
||||
default=0, metadata={"help": "force stop training at specified update"}
|
||||
)
|
||||
stop_time_hours: float = field(
|
||||
default=0,
|
||||
metadata={
|
||||
"help": "force stop training after specified cumulative time (if >0)"
|
||||
},
|
||||
)
|
||||
clip_norm: float = field(
|
||||
default=0.0, metadata={"help": "clip threshold of gradients"}
|
||||
)
|
||||
sentence_avg: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "normalize gradients by the number of sentences in a batch"
|
||||
" (default is to normalize by number of tokens)"
|
||||
},
|
||||
)
|
||||
update_freq: List[int] = field(
|
||||
default_factory=lambda: [1],
|
||||
metadata={"help": "update parameters every N_i batches, when in epoch i"},
|
||||
)
|
||||
lr: List[float] = field(
|
||||
default_factory=lambda: [0.25],
|
||||
metadata={
|
||||
"help": "learning rate for the first N epochs; all epochs >N using LR_N"
|
||||
" (note: this may be interpreted differently depending on --lr-scheduler)"
|
||||
},
|
||||
)
|
||||
stop_min_lr: float = field(
|
||||
default=-1.0,
|
||||
metadata={"help": "stop training when the learning rate reaches this minimum"},
|
||||
)
|
||||
use_bmuf: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "specify global optimizer for syncing models on different GPUs/shards"
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CheckpointConfig(FairseqDataclass):
|
||||
save_dir: str = field(
|
||||
default="checkpoints", metadata={"help": "path to save checkpoints"}
|
||||
)
|
||||
restore_file: str = field(
|
||||
default="checkpoint_last.pt",
|
||||
metadata={
|
||||
"help": "filename from which to load checkpoint "
|
||||
"(default: <save-dir>/checkpoint_last.pt"
|
||||
},
|
||||
)
|
||||
finetune_from_model: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "finetune from a pretrained model; note that meters and lr scheduler will be reset"
|
||||
},
|
||||
)
|
||||
reset_dataloader: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "if set, does not reload dataloader state from the checkpoint"
|
||||
},
|
||||
)
|
||||
reset_lr_scheduler: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "if set, does not load lr scheduler state from the checkpoint"
|
||||
},
|
||||
)
|
||||
reset_meters: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "if set, does not load meters from the checkpoint"},
|
||||
)
|
||||
reset_optimizer: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "if set, does not load optimizer state from the checkpoint"},
|
||||
)
|
||||
optimizer_overrides: str = field(
|
||||
default="{}",
|
||||
metadata={
|
||||
"help": "a dictionary used to override optimizer args when loading a checkpoint"
|
||||
},
|
||||
)
|
||||
save_interval: int = field(
|
||||
default=1, metadata={"help": "save a checkpoint every N epochs"}
|
||||
)
|
||||
save_interval_updates: int = field(
|
||||
default=0, metadata={"help": "save a checkpoint (and validate) every N updates"}
|
||||
)
|
||||
keep_interval_updates: int = field(
|
||||
default=-1,
|
||||
metadata={
|
||||
"help": "keep the last N checkpoints saved with --save-interval-updates"
|
||||
},
|
||||
)
|
||||
keep_last_epochs: int = field(
|
||||
default=-1, metadata={"help": "keep last N epoch checkpoints"}
|
||||
)
|
||||
keep_best_checkpoints: int = field(
|
||||
default=-1, metadata={"help": "keep best N checkpoints based on scores"}
|
||||
)
|
||||
no_save: bool = field(
|
||||
default=False, metadata={"help": "don't save models or checkpoints"}
|
||||
)
|
||||
no_epoch_checkpoints: bool = field(
|
||||
default=False, metadata={"help": "only store last and best checkpoints"}
|
||||
)
|
||||
no_last_checkpoints: bool = field(
|
||||
default=False, metadata={"help": "don't store last checkpoints"}
|
||||
)
|
||||
no_save_optimizer_state: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "don't save optimizer-state as part of checkpoint"},
|
||||
)
|
||||
best_checkpoint_metric: str = field(
|
||||
default="loss", metadata={"help": 'metric to use for saving "best" checkpoints'}
|
||||
)
|
||||
maximize_best_checkpoint_metric: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": 'select the largest metric value for saving "best" checkpoints'
|
||||
},
|
||||
)
|
||||
patience: int = field(
|
||||
default=-1,
|
||||
metadata={
|
||||
"help": (
|
||||
"early stop training if valid performance doesn't "
|
||||
"improve for N consecutive validation runs; note "
|
||||
"that this is influenced by --validate-interval"
|
||||
)
|
||||
},
|
||||
)
|
||||
checkpoint_suffix: str = field(
|
||||
default="", metadata={"help": "suffix to add to the checkpoint file name"}
|
||||
)
|
||||
checkpoint_shard_count: int = field(
|
||||
default=1,
|
||||
metadata={
|
||||
"help": "Number of shards containing the checkpoint - "
|
||||
"if the checkpoint is over 300GB, it is preferable "
|
||||
"to split it into shards to prevent OOM on CPU while loading "
|
||||
"the checkpoint"
|
||||
},
|
||||
)
|
||||
load_checkpoint_on_all_dp_ranks: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "load checkpoints on all data parallel devices "
|
||||
"(default: only load on rank 0 and broadcast to other devices)"
|
||||
},
|
||||
)
|
||||
model_parallel_size: int = II("common.model_parallel_size")
|
||||
distributed_rank: int = II("distributed_training.distributed_rank")
|
||||
|
||||
|
||||
@dataclass
|
||||
class FairseqBMUFConfig(FairseqDataclass):
|
||||
block_lr: float = field(
|
||||
default=1, metadata={"help": "block learning rate for bmuf"}
|
||||
)
|
||||
block_momentum: float = field(
|
||||
default=0.875, metadata={"help": "block momentum for bmuf"}
|
||||
)
|
||||
global_sync_iter: int = field(
|
||||
default=50, metadata={"help": "Iteration for syncing global model"}
|
||||
)
|
||||
warmup_iterations: int = field(
|
||||
default=500, metadata={"help": "warmup iterations for model to broadcast"}
|
||||
)
|
||||
use_nbm: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "Specify whether you want to use classical BM / Nesterov BM"},
|
||||
)
|
||||
average_sync: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "Specify whether you want to average the local momentum after each sync"
|
||||
},
|
||||
)
|
||||
distributed_world_size: int = II("distributed_training.distributed_world_size")
|
||||
|
||||
|
||||
@dataclass
|
||||
class GenerationConfig(FairseqDataclass):
|
||||
beam: int = field(
|
||||
default=5,
|
||||
metadata={"help": "beam size"},
|
||||
)
|
||||
nbest: int = field(
|
||||
default=1,
|
||||
metadata={"help": "number of hypotheses to output"},
|
||||
)
|
||||
max_len_a: float = field(
|
||||
default=0,
|
||||
metadata={
|
||||
"help": "generate sequences of maximum length ax + b, where x is the source length"
|
||||
},
|
||||
)
|
||||
max_len_b: int = field(
|
||||
default=200,
|
||||
metadata={
|
||||
"help": "generate sequences of maximum length ax + b, where x is the source length"
|
||||
},
|
||||
)
|
||||
min_len: int = field(
|
||||
default=1,
|
||||
metadata={"help": "minimum generation length"},
|
||||
)
|
||||
match_source_len: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "generations should match the source length"},
|
||||
)
|
||||
unnormalized: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "compare unnormalized hypothesis scores"},
|
||||
)
|
||||
no_early_stop: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "deprecated"},
|
||||
)
|
||||
no_beamable_mm: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "don't use BeamableMM in attention layers"},
|
||||
)
|
||||
lenpen: float = field(
|
||||
default=1,
|
||||
metadata={
|
||||
"help": "length penalty: <1.0 favors shorter, >1.0 favors longer sentences"
|
||||
},
|
||||
)
|
||||
unkpen: float = field(
|
||||
default=0,
|
||||
metadata={
|
||||
"help": "unknown word penalty: <0 produces more unks, >0 produces fewer"
|
||||
},
|
||||
)
|
||||
replace_unk: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "perform unknown replacement (optionally with alignment dictionary)",
|
||||
"argparse_const": "@@ ",
|
||||
},
|
||||
)
|
||||
sacrebleu: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "score with sacrebleu"},
|
||||
)
|
||||
score_reference: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "just score the reference translation"},
|
||||
)
|
||||
prefix_size: int = field(
|
||||
default=0,
|
||||
metadata={"help": "initialize generation by target prefix of given length"},
|
||||
)
|
||||
no_repeat_ngram_size: int = field(
|
||||
default=0,
|
||||
metadata={
|
||||
"help": "ngram blocking such that this size ngram cannot be repeated in the generation"
|
||||
},
|
||||
)
|
||||
sampling: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "sample hypotheses instead of using beam search"},
|
||||
)
|
||||
sampling_topk: int = field(
|
||||
default=-1,
|
||||
metadata={"help": "sample from top K likely next words instead of all words"},
|
||||
)
|
||||
sampling_topp: float = field(
|
||||
default=-1.0,
|
||||
metadata={
|
||||
"help": "sample from the smallest set whose cumulative probability mass exceeds p for next words"
|
||||
},
|
||||
)
|
||||
constraints: Optional[GENERATION_CONSTRAINTS_CHOICES] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "enables lexically constrained decoding",
|
||||
"argparse_const": "ordered",
|
||||
},
|
||||
)
|
||||
temperature: float = field(
|
||||
default=1.0,
|
||||
metadata={"help": "temperature for generation"},
|
||||
)
|
||||
diverse_beam_groups: int = field(
|
||||
default=-1,
|
||||
metadata={"help": "number of groups for Diverse Beam Search"},
|
||||
)
|
||||
diverse_beam_strength: float = field(
|
||||
default=0.5,
|
||||
metadata={"help": "strength of diversity penalty for Diverse Beam Search"},
|
||||
)
|
||||
diversity_rate: float = field(
|
||||
default=-1.0,
|
||||
metadata={"help": "strength of diversity penalty for Diverse Siblings Search"},
|
||||
)
|
||||
print_alignment: Optional[PRINT_ALIGNMENT_CHOICES] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "if set, uses attention feedback to compute and print alignment to source tokens "
|
||||
"(valid options are: hard, soft, otherwise treated as hard alignment)",
|
||||
"argparse_const": "hard",
|
||||
},
|
||||
)
|
||||
print_step: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "print steps"},
|
||||
)
|
||||
lm_path: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "path to lm checkpoint for lm fusion"},
|
||||
)
|
||||
lm_weight: float = field(
|
||||
default=0.0,
|
||||
metadata={"help": "weight for lm probs for lm fusion"},
|
||||
)
|
||||
|
||||
# arguments for iterative refinement generator
|
||||
iter_decode_eos_penalty: float = field(
|
||||
default=0.0,
|
||||
metadata={"help": "if > 0.0, it penalized early-stopping in decoding."},
|
||||
)
|
||||
iter_decode_max_iter: int = field(
|
||||
default=10,
|
||||
metadata={"help": "maximum iterations for iterative refinement."},
|
||||
)
|
||||
iter_decode_force_max_iter: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "if set, run exact the maximum number of iterations without early stop"
|
||||
},
|
||||
)
|
||||
iter_decode_with_beam: int = field(
|
||||
default=1,
|
||||
metadata={
|
||||
"help": "if > 1, model will generate translations varying by the lengths."
|
||||
},
|
||||
)
|
||||
iter_decode_with_external_reranker: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "if set, the last checkpoint are assumed to be a reranker to rescore the translations"
|
||||
},
|
||||
)
|
||||
retain_iter_history: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "if set, decoding returns the whole history of iterative refinement"
|
||||
},
|
||||
)
|
||||
retain_dropout: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "Use dropout at inference time"},
|
||||
)
|
||||
# temporarily set to Any until https://github.com/facebookresearch/hydra/issues/1117 is fixed
|
||||
# retain_dropout_modules: Optional[List[str]] = field(
|
||||
retain_dropout_modules: Any = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": "if set, only retain dropout for the specified modules; "
|
||||
"if not set, then dropout will be retained for all modules"
|
||||
},
|
||||
)
|
||||
# special decoding format for advanced decoding.
|
||||
decoding_format: Optional[GENERATION_DECODING_FORMAT_CHOICES] = field(
|
||||
default=None,
|
||||
metadata={"help": "special decoding format for advanced decoding."},
|
||||
)
|
||||
no_seed_provided: bool = field(
|
||||
default=False,
|
||||
metadata={"help": "if set, dont use seed for initializing random generators"},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CommonEvalConfig(FairseqDataclass):
|
||||
path: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={"help": "path(s) to model file(s), colon separated"},
|
||||
)
|
||||
post_process: Optional[str] = field(
|
||||
default=None,
|
||||
metadata={
|
||||
"help": (
|
||||
"post-process text by removing BPE, letter segmentation, etc. "
|
||||
"Valid options can be found in fairseq.data.utils.post_process."
|
||||
),
|
||||
"argparse_const": "subword_nmt",
|
||||
"argparse_alias": "--remove-bpe",
|
||||
},
|
||||
)
|
||||
quiet: bool = field(default=False, metadata={"help": "only print final scores"})
|
||||
model_overrides: str = field(
|
||||
default="{}",
|
||||
metadata={
|
||||
"help": "a dictionary used to override model args at generation that were used during model training"
|
||||
},
|
||||
)
|
||||
results_path: Optional[str] = field(
|
||||
default=None, metadata={"help": "path to save eval results (optional)"}
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class EvalLMConfig(FairseqDataclass):
|
||||
output_word_probs: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "if set, outputs words and their predicted log probabilities to standard output"
|
||||
},
|
||||
)
|
||||
output_word_stats: bool = field(
|
||||
default=False,
|
||||
metadata={
|
||||
"help": "if set, outputs word statistics such as word count, average probability, etc"
|
||||
},
|
||||
)
|
||||
context_window: int = field(
|
||||
default=0,
|
||||
metadata={
|
||||
"help": "ensures that every evaluated token has access to a context of at least this size, if possible"
|
||||
},
|
||||
)
|
||||
softmax_batch: int = field(
|
||||
default=sys.maxsize,
|
||||
metadata={
|
||||
"help": "if BxT is more than this, will batch the softmax over vocab to this amount of tokens, in order to fit into GPU memory"
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class InteractiveConfig(FairseqDataclass):
|
||||
buffer_size: int = field(
|
||||
default=0,
|
||||
metadata={
|
||||
"help": "read this many sentences into a buffer before processing them"
|
||||
},
|
||||
)
|
||||
input: str = field(
|
||||
default="-",
|
||||
metadata={"help": "file to read from; use - for stdin"},
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class FairseqConfig(FairseqDataclass):
|
||||
common: CommonConfig = CommonConfig()
|
||||
common_eval: CommonEvalConfig = CommonEvalConfig()
|
||||
distributed_training: DistributedTrainingConfig = DistributedTrainingConfig()
|
||||
dataset: DatasetConfig = DatasetConfig()
|
||||
optimization: OptimizationConfig = OptimizationConfig()
|
||||
checkpoint: CheckpointConfig = CheckpointConfig()
|
||||
bmuf: FairseqBMUFConfig = FairseqBMUFConfig()
|
||||
generation: GenerationConfig = GenerationConfig()
|
||||
eval_lm: EvalLMConfig = EvalLMConfig()
|
||||
interactive: InteractiveConfig = InteractiveConfig()
|
||||
model: Any = MISSING
|
||||
task: Any = None
|
||||
criterion: Any = None
|
||||
optimizer: Any = None
|
||||
lr_scheduler: Any = None
|
||||
scoring: Any = None
|
||||
bpe: Any = None
|
||||
tokenizer: Any = None
|
||||
@@ -0,0 +1,52 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
from enum import Enum, EnumMeta
|
||||
from typing import List
|
||||
|
||||
|
||||
class StrEnumMeta(EnumMeta):
|
||||
# this is workaround for submitit pickling leading to instance checks failing in hydra for StrEnum, see
|
||||
# https://github.com/facebookresearch/hydra/issues/1156
|
||||
@classmethod
|
||||
def __instancecheck__(cls, other):
|
||||
return "enum" in str(type(other))
|
||||
|
||||
|
||||
class StrEnum(Enum, metaclass=StrEnumMeta):
|
||||
def __str__(self):
|
||||
return self.value
|
||||
|
||||
def __eq__(self, other: str):
|
||||
return self.value == other
|
||||
|
||||
def __repr__(self):
|
||||
return self.value
|
||||
|
||||
def __hash__(self):
|
||||
return hash(str(self))
|
||||
|
||||
|
||||
def ChoiceEnum(choices: List[str]):
|
||||
"""return the Enum class used to enforce list of choices"""
|
||||
return StrEnum("Choices", {k: k for k in choices})
|
||||
|
||||
|
||||
LOG_FORMAT_CHOICES = ChoiceEnum(["json", "none", "simple", "tqdm"])
|
||||
DDP_BACKEND_CHOICES = ChoiceEnum([
|
||||
"c10d", # alias for pytorch_ddp
|
||||
"legacy_ddp",
|
||||
"no_c10d", # alias for legacy_ddp
|
||||
"pytorch_ddp",
|
||||
"slow_mo",
|
||||
])
|
||||
DATASET_IMPL_CHOICES = ChoiceEnum(["raw", "lazy", "cached", "mmap", "fasta"])
|
||||
GENERATION_CONSTRAINTS_CHOICES = ChoiceEnum(["ordered", "unordered"])
|
||||
GENERATION_DECODING_FORMAT_CHOICES = ChoiceEnum(
|
||||
["unigram", "ensemble", "vote", "dp", "bs"]
|
||||
)
|
||||
ZERO_SHARDING_CHOICES = ChoiceEnum(["none", "os"])
|
||||
PIPELINE_CHECKPOINT_CHOICES = ChoiceEnum(["always", "never", "except_last"])
|
||||
PRINT_ALIGNMENT_CHOICES = ChoiceEnum(["hard", "soft"])
|
||||
@@ -0,0 +1,61 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
"""isort:skip_file"""
|
||||
|
||||
import logging
|
||||
from hydra.core.config_store import ConfigStore
|
||||
from fairseq.dataclass.configs import FairseqConfig
|
||||
from omegaconf import DictConfig, OmegaConf
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def hydra_init(cfg_name="config") -> None:
|
||||
|
||||
cs = ConfigStore.instance()
|
||||
cs.store(name=cfg_name, node=FairseqConfig)
|
||||
|
||||
for k in FairseqConfig.__dataclass_fields__:
|
||||
v = FairseqConfig.__dataclass_fields__[k].default
|
||||
try:
|
||||
cs.store(name=k, node=v)
|
||||
except BaseException:
|
||||
logger.error(f"{k} - {v}")
|
||||
raise
|
||||
|
||||
|
||||
def add_defaults(cfg: DictConfig) -> None:
|
||||
"""This function adds default values that are stored in dataclasses that hydra doesn't know about """
|
||||
|
||||
from fairseq.registry import REGISTRIES
|
||||
from fairseq.tasks import TASK_DATACLASS_REGISTRY
|
||||
from fairseq.models import ARCH_MODEL_NAME_REGISTRY, MODEL_DATACLASS_REGISTRY
|
||||
from fairseq.dataclass.utils import merge_with_parent
|
||||
from typing import Any
|
||||
|
||||
OmegaConf.set_struct(cfg, False)
|
||||
|
||||
for k, v in FairseqConfig.__dataclass_fields__.items():
|
||||
field_cfg = cfg.get(k)
|
||||
if field_cfg is not None and v.type == Any:
|
||||
dc = None
|
||||
|
||||
if isinstance(field_cfg, str):
|
||||
field_cfg = DictConfig({"_name": field_cfg})
|
||||
field_cfg.__dict__["_parent"] = field_cfg.__dict__["_parent"]
|
||||
|
||||
name = field_cfg.get("_name")
|
||||
|
||||
if k == "task":
|
||||
dc = TASK_DATACLASS_REGISTRY.get(name)
|
||||
elif k == "model":
|
||||
name = ARCH_MODEL_NAME_REGISTRY.get(name, name)
|
||||
dc = MODEL_DATACLASS_REGISTRY.get(name)
|
||||
elif k in REGISTRIES:
|
||||
dc = REGISTRIES[k]["dataclass_registry"].get(name)
|
||||
|
||||
if dc is not None:
|
||||
cfg[k] = merge_with_parent(dc, field_cfg)
|
||||
@@ -0,0 +1,460 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import ast
|
||||
import inspect
|
||||
import logging
|
||||
import os
|
||||
import re
|
||||
from argparse import ArgumentError, ArgumentParser, Namespace
|
||||
from dataclasses import _MISSING_TYPE, MISSING
|
||||
from enum import Enum
|
||||
from typing import Any, Dict, List, Optional, Tuple, Type
|
||||
|
||||
from fairseq.dataclass import FairseqDataclass
|
||||
from fairseq.dataclass.configs import FairseqConfig
|
||||
from hydra.core.global_hydra import GlobalHydra
|
||||
from hydra.experimental import compose, initialize
|
||||
from omegaconf import DictConfig, OmegaConf, open_dict
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
def eval_str_list(x, x_type=float):
|
||||
if x is None:
|
||||
return None
|
||||
if isinstance(x, str):
|
||||
if len(x) == 0:
|
||||
return []
|
||||
x = ast.literal_eval(x)
|
||||
try:
|
||||
return list(map(x_type, x))
|
||||
except TypeError:
|
||||
return [x_type(x)]
|
||||
|
||||
|
||||
def interpret_dc_type(field_type):
|
||||
if isinstance(field_type, str):
|
||||
raise RuntimeError("field should be a type")
|
||||
|
||||
if field_type == Any:
|
||||
return str
|
||||
|
||||
typestring = str(field_type)
|
||||
if re.match(r"(typing.|^)Union\[(.*), NoneType\]$", typestring) or typestring.startswith("typing.Optional"):
|
||||
return field_type.__args__[0]
|
||||
return field_type
|
||||
|
||||
|
||||
def gen_parser_from_dataclass(
|
||||
parser: ArgumentParser,
|
||||
dataclass_instance: FairseqDataclass,
|
||||
delete_default: bool = False,
|
||||
) -> None:
|
||||
"""convert a dataclass instance to tailing parser arguments"""
|
||||
|
||||
def argparse_name(name: str):
|
||||
if name == "data":
|
||||
# normally data is positional args
|
||||
return name
|
||||
if name == "_name":
|
||||
# private member, skip
|
||||
return None
|
||||
return "--" + name.replace("_", "-")
|
||||
|
||||
def get_kwargs_from_dc(
|
||||
dataclass_instance: FairseqDataclass, k: str
|
||||
) -> Dict[str, Any]:
|
||||
"""k: dataclass attributes"""
|
||||
|
||||
kwargs = {}
|
||||
|
||||
field_type = dataclass_instance._get_type(k)
|
||||
inter_type = interpret_dc_type(field_type)
|
||||
|
||||
field_default = dataclass_instance._get_default(k)
|
||||
|
||||
if isinstance(inter_type, type) and issubclass(inter_type, Enum):
|
||||
field_choices = [t.value for t in list(inter_type)]
|
||||
else:
|
||||
field_choices = None
|
||||
|
||||
field_help = dataclass_instance._get_help(k)
|
||||
field_const = dataclass_instance._get_argparse_const(k)
|
||||
|
||||
if isinstance(field_default, str) and field_default.startswith("${"):
|
||||
kwargs["default"] = field_default
|
||||
else:
|
||||
if field_default is MISSING:
|
||||
kwargs["required"] = True
|
||||
if field_choices is not None:
|
||||
kwargs["choices"] = field_choices
|
||||
if (
|
||||
isinstance(inter_type, type)
|
||||
and (issubclass(inter_type, List) or issubclass(inter_type, Tuple))
|
||||
) or ("List" in str(inter_type) or "Tuple" in str(inter_type)):
|
||||
if "int" in str(inter_type):
|
||||
kwargs["type"] = lambda x: eval_str_list(x, int)
|
||||
elif "float" in str(inter_type):
|
||||
kwargs["type"] = lambda x: eval_str_list(x, float)
|
||||
elif "str" in str(inter_type):
|
||||
kwargs["type"] = lambda x: eval_str_list(x, str)
|
||||
else:
|
||||
raise NotImplementedError(
|
||||
"parsing of type " + str(inter_type) + " is not implemented"
|
||||
)
|
||||
if field_default is not MISSING:
|
||||
kwargs["default"] = (
|
||||
",".join(map(str, field_default))
|
||||
if field_default is not None
|
||||
else None
|
||||
)
|
||||
elif (
|
||||
isinstance(inter_type, type) and issubclass(inter_type, Enum)
|
||||
) or "Enum" in str(inter_type):
|
||||
kwargs["type"] = str
|
||||
if field_default is not MISSING:
|
||||
if isinstance(field_default, Enum):
|
||||
kwargs["default"] = field_default.value
|
||||
else:
|
||||
kwargs["default"] = field_default
|
||||
elif inter_type is bool:
|
||||
kwargs["action"] = (
|
||||
"store_false" if field_default is True else "store_true"
|
||||
)
|
||||
kwargs["default"] = field_default
|
||||
else:
|
||||
kwargs["type"] = inter_type
|
||||
if field_default is not MISSING:
|
||||
kwargs["default"] = field_default
|
||||
|
||||
kwargs["help"] = field_help
|
||||
if field_const is not None:
|
||||
kwargs["const"] = field_const
|
||||
kwargs["nargs"] = "?"
|
||||
|
||||
return kwargs
|
||||
|
||||
for k in dataclass_instance._get_all_attributes():
|
||||
field_name = argparse_name(dataclass_instance._get_name(k))
|
||||
field_type = dataclass_instance._get_type(k)
|
||||
if field_name is None:
|
||||
continue
|
||||
elif inspect.isclass(field_type) and issubclass(field_type, FairseqDataclass):
|
||||
gen_parser_from_dataclass(parser, field_type(), delete_default)
|
||||
continue
|
||||
|
||||
kwargs = get_kwargs_from_dc(dataclass_instance, k)
|
||||
|
||||
field_args = [field_name]
|
||||
alias = dataclass_instance._get_argparse_alias(k)
|
||||
if alias is not None:
|
||||
field_args.append(alias)
|
||||
|
||||
if "default" in kwargs:
|
||||
if isinstance(kwargs["default"], str) and kwargs["default"].startswith(
|
||||
"${"
|
||||
):
|
||||
if kwargs["help"] is None:
|
||||
# this is a field with a name that will be added elsewhere
|
||||
continue
|
||||
else:
|
||||
del kwargs["default"]
|
||||
if delete_default and "default" in kwargs:
|
||||
del kwargs["default"]
|
||||
try:
|
||||
parser.add_argument(*field_args, **kwargs)
|
||||
except ArgumentError:
|
||||
pass
|
||||
|
||||
|
||||
def _set_legacy_defaults(args, cls):
|
||||
"""Helper to set default arguments based on *add_args*."""
|
||||
if not hasattr(cls, "add_args"):
|
||||
return
|
||||
|
||||
import argparse
|
||||
|
||||
parser = argparse.ArgumentParser(
|
||||
argument_default=argparse.SUPPRESS, allow_abbrev=False
|
||||
)
|
||||
cls.add_args(parser)
|
||||
# copied from argparse.py:
|
||||
defaults = argparse.Namespace()
|
||||
for action in parser._actions:
|
||||
if action.dest is not argparse.SUPPRESS:
|
||||
if not hasattr(defaults, action.dest):
|
||||
if action.default is not argparse.SUPPRESS:
|
||||
setattr(defaults, action.dest, action.default)
|
||||
for key, default_value in vars(defaults).items():
|
||||
if not hasattr(args, key):
|
||||
setattr(args, key, default_value)
|
||||
|
||||
|
||||
def _override_attr(
|
||||
sub_node: str, data_class: Type[FairseqDataclass], args: Namespace
|
||||
) -> List[str]:
|
||||
overrides = []
|
||||
|
||||
if not inspect.isclass(data_class) or not issubclass(data_class, FairseqDataclass):
|
||||
return overrides
|
||||
|
||||
def get_default(f):
|
||||
if not isinstance(f.default_factory, _MISSING_TYPE):
|
||||
return f.default_factory()
|
||||
return f.default
|
||||
|
||||
for k, v in data_class.__dataclass_fields__.items():
|
||||
if k.startswith("_"):
|
||||
# private member, skip
|
||||
continue
|
||||
|
||||
val = get_default(v) if not hasattr(args, k) else getattr(args, k)
|
||||
|
||||
field_type = interpret_dc_type(v.type)
|
||||
if (
|
||||
isinstance(val, str)
|
||||
and not val.startswith("${") # not interpolation
|
||||
and field_type != str
|
||||
and (
|
||||
not inspect.isclass(field_type) or not issubclass(field_type, Enum)
|
||||
) # not choices enum
|
||||
):
|
||||
# upgrade old models that stored complex parameters as string
|
||||
val = ast.literal_eval(val)
|
||||
|
||||
if isinstance(val, tuple):
|
||||
val = list(val)
|
||||
|
||||
v_type = getattr(v.type, "__origin__", None)
|
||||
if (
|
||||
(v_type is List or v_type is list or v_type is Optional)
|
||||
# skip interpolation
|
||||
and not (isinstance(val, str) and val.startswith("${"))
|
||||
):
|
||||
# if type is int but val is float, then we will crash later - try to convert here
|
||||
if hasattr(v.type, '__args__'):
|
||||
t_args = v.type.__args__
|
||||
if len(t_args) == 1:
|
||||
val = list(map(t_args[0], val))
|
||||
elif val is not None and (field_type is int or field_type is bool or field_type is float):
|
||||
try:
|
||||
val = field_type(val)
|
||||
except:
|
||||
pass # ignore errors here, they are often from interpolation args
|
||||
|
||||
if val is None:
|
||||
overrides.append("{}.{}=null".format(sub_node, k))
|
||||
elif val == "":
|
||||
overrides.append("{}.{}=''".format(sub_node, k))
|
||||
elif isinstance(val, str):
|
||||
val = val.replace("'", r"\'")
|
||||
overrides.append("{}.{}='{}'".format(sub_node, k, val))
|
||||
elif isinstance(val, FairseqDataclass):
|
||||
overrides += _override_attr(f"{sub_node}.{k}", type(val), args)
|
||||
elif isinstance(val, Namespace):
|
||||
sub_overrides, _ = override_module_args(val)
|
||||
for so in sub_overrides:
|
||||
overrides.append(f"{sub_node}.{k}.{so}")
|
||||
else:
|
||||
overrides.append("{}.{}={}".format(sub_node, k, val))
|
||||
|
||||
return overrides
|
||||
|
||||
|
||||
def migrate_registry(
|
||||
name, value, registry, args, overrides, deletes, use_name_as_val=False
|
||||
):
|
||||
if value in registry:
|
||||
overrides.append("{}={}".format(name, value))
|
||||
overrides.append("{}._name={}".format(name, value))
|
||||
overrides.extend(_override_attr(name, registry[value], args))
|
||||
elif use_name_as_val and value is not None:
|
||||
overrides.append("{}={}".format(name, value))
|
||||
else:
|
||||
deletes.append(name)
|
||||
|
||||
|
||||
def override_module_args(args: Namespace) -> Tuple[List[str], List[str]]:
|
||||
"""use the field in args to overrides those in cfg"""
|
||||
overrides = []
|
||||
deletes = []
|
||||
|
||||
for k in FairseqConfig.__dataclass_fields__.keys():
|
||||
overrides.extend(
|
||||
_override_attr(k, FairseqConfig.__dataclass_fields__[k].type, args)
|
||||
)
|
||||
|
||||
if args is not None:
|
||||
if hasattr(args, "task"):
|
||||
from fairseq.tasks import TASK_DATACLASS_REGISTRY
|
||||
|
||||
migrate_registry(
|
||||
"task", args.task, TASK_DATACLASS_REGISTRY, args, overrides, deletes
|
||||
)
|
||||
else:
|
||||
deletes.append("task")
|
||||
|
||||
# these options will be set to "None" if they have not yet been migrated
|
||||
# so we can populate them with the entire flat args
|
||||
CORE_REGISTRIES = {"criterion", "optimizer", "lr_scheduler"}
|
||||
|
||||
from fairseq.registry import REGISTRIES
|
||||
|
||||
for k, v in REGISTRIES.items():
|
||||
if hasattr(args, k):
|
||||
migrate_registry(
|
||||
k,
|
||||
getattr(args, k),
|
||||
v["dataclass_registry"],
|
||||
args,
|
||||
overrides,
|
||||
deletes,
|
||||
use_name_as_val=k not in CORE_REGISTRIES,
|
||||
)
|
||||
else:
|
||||
deletes.append(k)
|
||||
|
||||
no_dc = True
|
||||
if hasattr(args, "arch"):
|
||||
from fairseq.models import ARCH_MODEL_REGISTRY, ARCH_MODEL_NAME_REGISTRY
|
||||
|
||||
if args.arch in ARCH_MODEL_REGISTRY:
|
||||
m_cls = ARCH_MODEL_REGISTRY[args.arch]
|
||||
dc = getattr(m_cls, "__dataclass", None)
|
||||
if dc is not None:
|
||||
m_name = ARCH_MODEL_NAME_REGISTRY[args.arch]
|
||||
overrides.append("model={}".format(m_name))
|
||||
overrides.append("model._name={}".format(args.arch))
|
||||
# override model params with those exist in args
|
||||
overrides.extend(_override_attr("model", dc, args))
|
||||
no_dc = False
|
||||
if no_dc:
|
||||
deletes.append("model")
|
||||
|
||||
return overrides, deletes
|
||||
|
||||
|
||||
def convert_namespace_to_omegaconf(args: Namespace) -> DictConfig:
|
||||
"""Convert a flat argparse.Namespace to a structured DictConfig."""
|
||||
|
||||
# Here we are using field values provided in args to override counterparts inside config object
|
||||
overrides, deletes = override_module_args(args)
|
||||
|
||||
# configs will be in fairseq/config after installation
|
||||
config_path = os.path.join("..", "config")
|
||||
|
||||
GlobalHydra.instance().clear()
|
||||
|
||||
with initialize(config_path=config_path):
|
||||
try:
|
||||
composed_cfg = compose("config", overrides=overrides, strict=False)
|
||||
except:
|
||||
logger.error("Error when composing. Overrides: " + str(overrides))
|
||||
raise
|
||||
|
||||
for k in deletes:
|
||||
composed_cfg[k] = None
|
||||
|
||||
cfg = OmegaConf.create(
|
||||
OmegaConf.to_container(composed_cfg, resolve=True, enum_to_str=True)
|
||||
)
|
||||
|
||||
# hack to be able to set Namespace in dict config. this should be removed when we update to newer
|
||||
# omegaconf version that supports object flags, or when we migrate all existing models
|
||||
from omegaconf import _utils
|
||||
|
||||
old_primitive = _utils.is_primitive_type
|
||||
_utils.is_primitive_type = lambda _: True
|
||||
|
||||
if cfg.task is None and getattr(args, "task", None):
|
||||
cfg.task = Namespace(**vars(args))
|
||||
from fairseq.tasks import TASK_REGISTRY
|
||||
|
||||
_set_legacy_defaults(cfg.task, TASK_REGISTRY[args.task])
|
||||
cfg.task._name = args.task
|
||||
if cfg.model is None and getattr(args, "arch", None):
|
||||
cfg.model = Namespace(**vars(args))
|
||||
from fairseq.models import ARCH_MODEL_REGISTRY
|
||||
|
||||
_set_legacy_defaults(cfg.model, ARCH_MODEL_REGISTRY[args.arch])
|
||||
cfg.model._name = args.arch
|
||||
if cfg.optimizer is None and getattr(args, "optimizer", None):
|
||||
cfg.optimizer = Namespace(**vars(args))
|
||||
from fairseq.optim import OPTIMIZER_REGISTRY
|
||||
|
||||
_set_legacy_defaults(cfg.optimizer, OPTIMIZER_REGISTRY[args.optimizer])
|
||||
cfg.optimizer._name = args.optimizer
|
||||
if cfg.lr_scheduler is None and getattr(args, "lr_scheduler", None):
|
||||
cfg.lr_scheduler = Namespace(**vars(args))
|
||||
from fairseq.optim.lr_scheduler import LR_SCHEDULER_REGISTRY
|
||||
|
||||
_set_legacy_defaults(cfg.lr_scheduler, LR_SCHEDULER_REGISTRY[args.lr_scheduler])
|
||||
cfg.lr_scheduler._name = args.lr_scheduler
|
||||
if cfg.criterion is None and getattr(args, "criterion", None):
|
||||
cfg.criterion = Namespace(**vars(args))
|
||||
from fairseq.criterions import CRITERION_REGISTRY
|
||||
|
||||
_set_legacy_defaults(cfg.criterion, CRITERION_REGISTRY[args.criterion])
|
||||
cfg.criterion._name = args.criterion
|
||||
|
||||
_utils.is_primitive_type = old_primitive
|
||||
OmegaConf.set_struct(cfg, True)
|
||||
return cfg
|
||||
|
||||
|
||||
def populate_dataclass(
|
||||
dataclass: FairseqDataclass,
|
||||
args: Namespace,
|
||||
) -> FairseqDataclass:
|
||||
for k in dataclass.__dataclass_fields__.keys():
|
||||
if k.startswith("_"):
|
||||
# private member, skip
|
||||
continue
|
||||
if hasattr(args, k):
|
||||
setattr(dataclass, k, getattr(args, k))
|
||||
|
||||
return dataclass
|
||||
|
||||
|
||||
def overwrite_args_by_name(cfg: DictConfig, overrides: Dict[str, any]):
|
||||
# this will be deprecated when we get rid of argparse and model_overrides logic
|
||||
|
||||
from fairseq.registry import REGISTRIES
|
||||
|
||||
with open_dict(cfg):
|
||||
for k in cfg.keys():
|
||||
# "k in cfg" will return false if its a "mandatory value (e.g. ???)"
|
||||
if k in cfg and isinstance(cfg[k], DictConfig):
|
||||
if k in overrides and isinstance(overrides[k], dict):
|
||||
for ok, ov in overrides[k].items():
|
||||
if isinstance(ov, dict):
|
||||
overwrite_args_by_name(cfg[k][ok], ov)
|
||||
else:
|
||||
cfg[k][ok] = ov
|
||||
else:
|
||||
overwrite_args_by_name(cfg[k], overrides)
|
||||
elif k in cfg and isinstance(cfg[k], Namespace):
|
||||
for override_key, val in overrides.items():
|
||||
setattr(cfg[k], override_key, val)
|
||||
elif k in overrides:
|
||||
if (
|
||||
k in REGISTRIES
|
||||
and overrides[k] in REGISTRIES[k]["dataclass_registry"]
|
||||
):
|
||||
cfg[k] = DictConfig(
|
||||
REGISTRIES[k]["dataclass_registry"][overrides[k]]
|
||||
)
|
||||
overwrite_args_by_name(cfg[k], overrides)
|
||||
cfg[k]._name = overrides[k]
|
||||
else:
|
||||
cfg[k] = overrides[k]
|
||||
|
||||
|
||||
def merge_with_parent(dc: FairseqDataclass, cfg: FairseqDataclass):
|
||||
merged_cfg = OmegaConf.merge(dc, cfg)
|
||||
merged_cfg.__dict__["_parent"] = cfg.__dict__["_parent"]
|
||||
OmegaConf.set_struct(merged_cfg, True)
|
||||
return merged_cfg
|
||||
Reference in New Issue
Block a user