chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,7 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
# import models/tasks to register them
|
||||
from . import dummy_lm, dummy_masked_lm, dummy_model, dummy_mt # noqa
|
||||
@@ -0,0 +1,118 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import logging
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Optional
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from fairseq.data import Dictionary, FairseqDataset
|
||||
from fairseq.dataclass import FairseqDataclass
|
||||
from fairseq.tasks import FairseqTask, register_task
|
||||
from omegaconf import II
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@dataclass
|
||||
class DummyLMConfig(FairseqDataclass):
|
||||
dict_size: int = 49996
|
||||
dataset_size: int = 100000
|
||||
tokens_per_sample: int = field(
|
||||
default=512, metadata={"help": "max sequence length"}
|
||||
)
|
||||
add_bos_token: bool = False
|
||||
batch_size: Optional[int] = II("dataset.batch_size")
|
||||
max_tokens: Optional[int] = II("dataset.max_tokens")
|
||||
max_target_positions: int = II("task.tokens_per_sample")
|
||||
|
||||
|
||||
@register_task("dummy_lm", dataclass=DummyLMConfig)
|
||||
class DummyLMTask(FairseqTask):
|
||||
|
||||
def __init__(self, cfg: DummyLMConfig):
|
||||
super().__init__(cfg)
|
||||
|
||||
# load dictionary
|
||||
self.dictionary = Dictionary()
|
||||
for i in range(cfg.dict_size):
|
||||
self.dictionary.add_symbol("word{}".format(i))
|
||||
self.dictionary.pad_to_multiple_(8) # often faster if divisible by 8
|
||||
logger.info("dictionary: {} types".format(len(self.dictionary)))
|
||||
|
||||
seq = torch.arange(cfg.tokens_per_sample + 1) + self.dictionary.pad() + 1
|
||||
|
||||
self.dummy_src = seq[:-1]
|
||||
self.dummy_tgt = seq[1:]
|
||||
|
||||
def load_dataset(self, split, epoch=1, combine=False, **kwargs):
|
||||
"""Load a given dataset split.
|
||||
Args:
|
||||
split (str): name of the split (e.g., train, valid, test)
|
||||
"""
|
||||
if self.cfg.batch_size is not None:
|
||||
bsz = self.cfg.batch_size
|
||||
else:
|
||||
bsz = max(1, self.cfg.max_tokens // self.cfg.tokens_per_sample)
|
||||
self.datasets[split] = DummyDataset(
|
||||
{
|
||||
"id": 1,
|
||||
"net_input": {
|
||||
"src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
|
||||
"src_lengths": torch.full(
|
||||
(bsz,), self.cfg.tokens_per_sample, dtype=torch.long
|
||||
),
|
||||
},
|
||||
"target": torch.stack([self.dummy_tgt for _ in range(bsz)]),
|
||||
"nsentences": bsz,
|
||||
"ntokens": bsz * self.cfg.tokens_per_sample,
|
||||
},
|
||||
num_items=self.cfg.dataset_size,
|
||||
item_size=self.cfg.tokens_per_sample,
|
||||
)
|
||||
|
||||
@property
|
||||
def source_dictionary(self):
|
||||
return self.dictionary
|
||||
|
||||
@property
|
||||
def target_dictionary(self):
|
||||
return self.dictionary
|
||||
|
||||
|
||||
class DummyDataset(FairseqDataset):
|
||||
def __init__(self, batch, num_items, item_size):
|
||||
super().__init__()
|
||||
self.batch = batch
|
||||
self.num_items = num_items
|
||||
self.item_size = item_size
|
||||
|
||||
def __getitem__(self, index):
|
||||
return index
|
||||
|
||||
def __len__(self):
|
||||
return self.num_items
|
||||
|
||||
def collater(self, samples):
|
||||
return self.batch
|
||||
|
||||
@property
|
||||
def sizes(self):
|
||||
return np.array([self.item_size] * self.num_items)
|
||||
|
||||
def num_tokens(self, index):
|
||||
return self.item_size
|
||||
|
||||
def size(self, index):
|
||||
return self.item_size
|
||||
|
||||
def ordered_indices(self):
|
||||
return np.arange(self.num_items)
|
||||
|
||||
@property
|
||||
def supports_prefetch(self):
|
||||
return False
|
||||
@@ -0,0 +1,127 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from fairseq.data import Dictionary, FairseqDataset
|
||||
from fairseq.tasks import LegacyFairseqTask, register_task
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@register_task("dummy_masked_lm")
|
||||
class DummyMaskedLMTask(LegacyFairseqTask):
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
"""Add task-specific arguments to the parser."""
|
||||
parser.add_argument("--dict-size", default=49995, type=int)
|
||||
parser.add_argument("--dataset-size", default=100000, type=int)
|
||||
parser.add_argument(
|
||||
"--tokens-per-sample",
|
||||
default=512,
|
||||
type=int,
|
||||
help="max number of total tokens over all segments "
|
||||
"per sample for BERT dataset",
|
||||
)
|
||||
|
||||
def __init__(self, args, dictionary):
|
||||
super().__init__(args)
|
||||
self.dictionary = dictionary
|
||||
|
||||
# add mask token
|
||||
self.mask_idx = dictionary.add_symbol("<mask>")
|
||||
dictionary.pad_to_multiple_(8) # often faster if divisible by 8
|
||||
|
||||
mask_idx = 0
|
||||
pad_idx = 1
|
||||
seq = torch.arange(args.tokens_per_sample) + pad_idx + 1
|
||||
mask = torch.arange(2, args.tokens_per_sample, 7) # ~15%
|
||||
src = seq.clone()
|
||||
src[mask] = mask_idx
|
||||
tgt = torch.full_like(seq, pad_idx)
|
||||
tgt[mask] = seq[mask]
|
||||
|
||||
self.dummy_src = src
|
||||
self.dummy_tgt = tgt
|
||||
|
||||
@classmethod
|
||||
def setup_task(cls, args, **kwargs):
|
||||
"""Setup the task. """
|
||||
dictionary = Dictionary()
|
||||
for i in range(args.dict_size):
|
||||
dictionary.add_symbol("word{}".format(i))
|
||||
logger.info("dictionary: {} types".format(len(dictionary)))
|
||||
return cls(args, dictionary)
|
||||
|
||||
def load_dataset(self, split, epoch=1, combine=False, **kwargs):
|
||||
"""Load a given dataset split.
|
||||
Args:
|
||||
split (str): name of the split (e.g., train, valid, test)
|
||||
"""
|
||||
if self.args.batch_size is not None:
|
||||
bsz = self.args.batch_size
|
||||
else:
|
||||
bsz = max(1, self.args.max_tokens // self.args.tokens_per_sample)
|
||||
self.datasets[split] = DummyDataset(
|
||||
{
|
||||
"id": 1,
|
||||
"net_input": {
|
||||
"src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
|
||||
"src_lengths": torch.full(
|
||||
(bsz,), self.args.tokens_per_sample, dtype=torch.long
|
||||
),
|
||||
},
|
||||
"target": torch.stack([self.dummy_tgt for _ in range(bsz)]),
|
||||
"nsentences": bsz,
|
||||
"ntokens": bsz * self.args.tokens_per_sample,
|
||||
},
|
||||
num_items=self.args.dataset_size,
|
||||
item_size=self.args.tokens_per_sample,
|
||||
)
|
||||
|
||||
@property
|
||||
def source_dictionary(self):
|
||||
return self.dictionary
|
||||
|
||||
@property
|
||||
def target_dictionary(self):
|
||||
return self.dictionary
|
||||
|
||||
|
||||
class DummyDataset(FairseqDataset):
|
||||
def __init__(self, batch, num_items, item_size):
|
||||
super().__init__()
|
||||
self.batch = batch
|
||||
self.num_items = num_items
|
||||
self.item_size = item_size
|
||||
|
||||
def __getitem__(self, index):
|
||||
return index
|
||||
|
||||
def __len__(self):
|
||||
return self.num_items
|
||||
|
||||
def collater(self, samples):
|
||||
return self.batch
|
||||
|
||||
@property
|
||||
def sizes(self):
|
||||
return np.array([self.item_size] * self.num_items)
|
||||
|
||||
def num_tokens(self, index):
|
||||
return self.item_size
|
||||
|
||||
def size(self, index):
|
||||
return self.item_size
|
||||
|
||||
def ordered_indices(self):
|
||||
return np.arange(self.num_items)
|
||||
|
||||
@property
|
||||
def supports_prefetch(self):
|
||||
return False
|
||||
@@ -0,0 +1,96 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import torch.nn as nn
|
||||
import torch.nn.functional as F
|
||||
from fairseq.data import Dictionary
|
||||
from fairseq.models import (
|
||||
FairseqDecoder,
|
||||
FairseqLanguageModel,
|
||||
register_model,
|
||||
register_model_architecture,
|
||||
)
|
||||
|
||||
|
||||
@register_model("dummy_model")
|
||||
class DummyModel(FairseqLanguageModel):
|
||||
def __init__(self, args, encoder):
|
||||
super().__init__(encoder)
|
||||
self.args = args
|
||||
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
parser.add_argument("--num-layers", type=int, default=24)
|
||||
parser.add_argument("--embed-dim", type=int, default=1024)
|
||||
|
||||
@classmethod
|
||||
def build_model(cls, args, task):
|
||||
encoder = DummyEncoder(
|
||||
num_embed=len(task.target_dictionary),
|
||||
embed_dim=args.embed_dim,
|
||||
num_layers=args.num_layers,
|
||||
)
|
||||
return cls(args, encoder)
|
||||
|
||||
def forward(self, src_tokens, masked_tokens=None, **kwargs):
|
||||
return self.decoder(src_tokens, masked_tokens=masked_tokens)
|
||||
|
||||
|
||||
class DummyEncoder(FairseqDecoder):
|
||||
def __init__(self, num_embed=50000, embed_dim=1024, num_layers=24):
|
||||
super().__init__(Dictionary())
|
||||
self.embed = nn.Embedding(
|
||||
num_embeddings=num_embed, embedding_dim=embed_dim, padding_idx=0
|
||||
)
|
||||
self.layers_a = nn.ModuleList(
|
||||
[
|
||||
nn.Sequential(
|
||||
nn.LayerNorm(embed_dim),
|
||||
nn.Linear(embed_dim, 3 * embed_dim), # q, k, v input projection
|
||||
nn.Linear(3 * embed_dim, embed_dim), # skip self-attention
|
||||
nn.Linear(embed_dim, embed_dim), # output projection
|
||||
nn.Dropout(),
|
||||
)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.layers_b = nn.ModuleList(
|
||||
[
|
||||
nn.Sequential(
|
||||
nn.LayerNorm(embed_dim),
|
||||
nn.Linear(embed_dim, 4 * embed_dim), # FFN
|
||||
nn.ReLU(),
|
||||
nn.Linear(4 * embed_dim, embed_dim), # FFN
|
||||
nn.Dropout(0.1),
|
||||
)
|
||||
for i in range(num_layers)
|
||||
]
|
||||
)
|
||||
self.out_proj = nn.Linear(embed_dim, num_embed)
|
||||
|
||||
def forward(self, tokens, masked_tokens=None):
|
||||
x = self.embed(tokens)
|
||||
for layer_a, layer_b in zip(self.layers_a, self.layers_b):
|
||||
x = x + layer_a(x)
|
||||
x = x + layer_b(x)
|
||||
x = self.out_proj(x)
|
||||
if masked_tokens is not None:
|
||||
x = x[masked_tokens]
|
||||
return (x,)
|
||||
|
||||
def max_positions(self):
|
||||
return 1024
|
||||
|
||||
def get_normalized_probs(self, net_output, log_probs, sample=None):
|
||||
logits = net_output[0].float()
|
||||
if log_probs:
|
||||
return F.log_softmax(logits, dim=-1)
|
||||
else:
|
||||
return F.softmax(logits, dim=-1)
|
||||
|
||||
|
||||
@register_model_architecture("dummy_model", "dummy_model")
|
||||
def base_architecture(args):
|
||||
pass
|
||||
@@ -0,0 +1,119 @@
|
||||
# Copyright (c) Facebook, Inc. and its affiliates.
|
||||
#
|
||||
# This source code is licensed under the MIT license found in the
|
||||
# LICENSE file in the root directory of this source tree.
|
||||
|
||||
import logging
|
||||
|
||||
import numpy as np
|
||||
import torch
|
||||
from fairseq.data import Dictionary, FairseqDataset
|
||||
from fairseq.tasks import LegacyFairseqTask, register_task
|
||||
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
|
||||
|
||||
@register_task("dummy_mt")
|
||||
class DummyMTTask(LegacyFairseqTask):
|
||||
@staticmethod
|
||||
def add_args(parser):
|
||||
"""Add task-specific arguments to the parser."""
|
||||
parser.add_argument("--dict-size", default=49996, type=int)
|
||||
parser.add_argument("--dataset-size", default=100000, type=int)
|
||||
parser.add_argument("--src-len", default=30, type=int)
|
||||
parser.add_argument("--tgt-len", default=30, type=int)
|
||||
|
||||
def __init__(self, args, dictionary):
|
||||
super().__init__(args)
|
||||
self.dictionary = dictionary
|
||||
self.seed = args.seed
|
||||
|
||||
dictionary.pad_to_multiple_(8) # often faster if divisible by 8
|
||||
|
||||
self.dummy_src = torch.arange(args.src_len + 1) + dictionary.pad() + 1
|
||||
self.dummy_tgt = torch.arange(args.tgt_len + 1) + dictionary.pad() + 1
|
||||
|
||||
@classmethod
|
||||
def setup_task(cls, args, **kwargs):
|
||||
"""Setup the task. """
|
||||
dictionary = Dictionary()
|
||||
for i in range(args.dict_size):
|
||||
dictionary.add_symbol("word{}".format(i))
|
||||
logger.info("dictionary: {} types".format(len(dictionary)))
|
||||
|
||||
args.max_source_positions = args.src_len + dictionary.pad() + 2
|
||||
args.max_target_positions = args.tgt_len + dictionary.pad() + 2
|
||||
|
||||
return cls(args, dictionary)
|
||||
|
||||
def load_dataset(self, split, epoch=1, combine=False, **kwargs):
|
||||
"""Load a given dataset split.
|
||||
Args:
|
||||
split (str): name of the split (e.g., train, valid, test)
|
||||
"""
|
||||
item_size = max(self.args.src_len, self.args.tgt_len)
|
||||
if self.args.batch_size is not None:
|
||||
bsz = self.args.batch_size
|
||||
else:
|
||||
bsz = max(1, self.args.max_tokens // item_size)
|
||||
tgt = torch.stack([self.dummy_tgt for _ in range(bsz)])
|
||||
self.datasets[split] = DummyDataset(
|
||||
{
|
||||
"id": 1,
|
||||
"net_input": {
|
||||
"src_tokens": torch.stack([self.dummy_src for _ in range(bsz)]),
|
||||
"src_lengths": torch.full(
|
||||
(bsz,), self.args.src_len, dtype=torch.long
|
||||
),
|
||||
"prev_output_tokens": tgt.clone(),
|
||||
},
|
||||
"target": tgt,
|
||||
"nsentences": bsz,
|
||||
"ntokens": bsz * self.args.tgt_len,
|
||||
},
|
||||
num_items=self.args.dataset_size,
|
||||
item_size=item_size,
|
||||
)
|
||||
|
||||
@property
|
||||
def source_dictionary(self):
|
||||
return self.dictionary
|
||||
|
||||
@property
|
||||
def target_dictionary(self):
|
||||
return self.dictionary
|
||||
|
||||
|
||||
class DummyDataset(FairseqDataset):
|
||||
def __init__(self, batch, num_items, item_size):
|
||||
super().__init__()
|
||||
self.batch = batch
|
||||
self.num_items = num_items
|
||||
self.item_size = item_size
|
||||
|
||||
def __getitem__(self, index):
|
||||
return index
|
||||
|
||||
def __len__(self):
|
||||
return self.num_items
|
||||
|
||||
def collater(self, samples):
|
||||
return self.batch
|
||||
|
||||
@property
|
||||
def sizes(self):
|
||||
return np.array([self.item_size] * self.num_items)
|
||||
|
||||
def num_tokens(self, index):
|
||||
return self.item_size
|
||||
|
||||
def size(self, index):
|
||||
return self.item_size
|
||||
|
||||
def ordered_indices(self):
|
||||
return np.arange(self.num_items)
|
||||
|
||||
@property
|
||||
def supports_prefetch(self):
|
||||
return False
|
||||
Reference in New Issue
Block a user