wehub/paddlepaddle--paddle
1
0
Fork 0
Code Issues Pull Requests Actions 2 Deployments Models Agent Notes Packages Projects Releases Wiki Activity

chore: import upstream snapshot with attribution

Browse Source
This commit is contained in:
wehub-resource-sync
2026-07-13 12:40:42 +08:00
commit e25996e7db
15472 changed files with 3536181 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
python/paddle/text/__init__.py
+36
View File
@@ -0,0 +1,36 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .datasets import (
WMT14,
WMT16,
Conll05st,
Imdb,
Imikolov,
Movielens,
UCIHousing,
)
from .viterbi_decode import ViterbiDecoder, viterbi_decode
__all__ = [
'Conll05st',
'Imdb',
'Imikolov',
'Movielens',
'UCIHousing',
'WMT14',
'WMT16',
'ViterbiDecoder',
'viterbi_decode',
]
python/paddle/text/datasets/__init__.py
+23
View File
@@ -0,0 +1,23 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from .conll05 import Conll05st # noqa: F401
from .imdb import Imdb # noqa: F401
from .imikolov import Imikolov # noqa: F401
from .movielens import Movielens # noqa: F401
from .uci_housing import UCIHousing # noqa: F401
from .wmt14 import WMT14 # noqa: F401
from .wmt16 import WMT16 # noqa: F401
__all__ = []
python/paddle/text/datasets/conll05.py
+399
View File
@@ -0,0 +1,399 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
if TYPE_CHECKING:
import numpy as np
import numpy.typing as npt
import gzip
import tarfile
import numpy as np
from paddle.dataset.common import _check_exists_and_download
from paddle.io import Dataset
__all__ = []
DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz'
DATA_MD5 = '387719152ae52d60422c016e92a742fc'
WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt'
WORDDICT_MD5 = 'ea7fb7d4c75cc6254716f0177a506baa'
VERBDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FverbDict.txt'
VERBDICT_MD5 = '0d2977293bbb6cbefab5b0f97db1e77c'
TRGDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FtargetDict.txt'
TRGDICT_MD5 = 'd8c7f03ceb5fc2e5a0fa7503a4353751'
EMB_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2Femb'
EMB_MD5 = 'bf436eb0faa1f6f9103017f8be57cdb7'
UNK_IDX = 0
class Conll05st(Dataset):
"""
This class implements the Conll05st test dataset. For details, please refer to the relevant documentation:https://aclanthology.org/W05-0620.pdf
Note: only support download test dataset automatically for that
only test dataset of Conll05st is public.
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None
word_dict_file(str|None): path to word dictionary file, can be set None if
:attr:`download` is True. Default None
verb_dict_file(str|None): path to verb dictionary file, can be set None if
:attr:`download` is True. Default None
target_dict_file(str|None): path to target dictionary file, can be set None if
:attr:`download` is True. Default None
emb_file(str|None): path to embedding dictionary file, only used for
:code:`get_embedding` can be set None if :attr:`download` is
True. Default None
download(bool): whether to download dataset automatically if
:attr:`data_file` :attr:`word_dict_file` :attr:`verb_dict_file`
:attr:`target_dict_file` is not set. Default True
Returns:
Dataset: instance of conll05st dataset
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.text.datasets import Conll05st
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, pred_idx, mark, label):
... return paddle.sum(pred_idx), paddle.sum(mark), paddle.sum(label)
>>> conll05st = Conll05st()
>>> for i in range(10):
... pred_idx, mark, label = conll05st[i][-3:]
... pred_idx = paddle.to_tensor(pred_idx)
... mark = paddle.to_tensor(mark)
... label = paddle.to_tensor(label)
...
... model = SimpleNet()
... pred_idx, mark, label = model(pred_idx, mark, label)
... print(pred_idx.item(), mark.item(), label.item())
>>> # doctest: +SKIP('label will change')
65840 5 1991
92560 5 3686
99120 5 457
121960 5 3945
4774 5 2378
14973 5 1938
36921 5 1090
26908 5 2329
62965 5 2968
97755 5 2674
"""
data_file: str | None
word_dict_file: str | None
verb_dict_file: str | None
target_dict_file: str | None
emb_file: str | None
word_dict: dict[str, int]
predicate_dict: dict[str, int]
label_dict: dict[str, int]
sentences: list
predicates: list
labels: list
def __init__(
self,
data_file: str | None = None,
word_dict_file: str | None = None,
verb_dict_file: str | None = None,
target_dict_file: str | None = None,
emb_file: str | None = None,
download: bool = True,
):
self.data_file = data_file
if self.data_file is None:
assert download, (
"data_file is not set and downloading automatically is disabled"
)
self.data_file = _check_exists_and_download(
data_file, DATA_URL, DATA_MD5, 'conll05st', download
)
self.word_dict_file = word_dict_file
if self.word_dict_file is None:
assert download, (
"word_dict_file is not set and downloading automatically is disabled"
)
self.word_dict_file = _check_exists_and_download(
word_dict_file,
WORDDICT_URL,
WORDDICT_MD5,
'conll05st',
download,
)
self.verb_dict_file = verb_dict_file
if self.verb_dict_file is None:
assert download, (
"verb_dict_file is not set and downloading automatically is disabled"
)
self.verb_dict_file = _check_exists_and_download(
verb_dict_file,
VERBDICT_URL,
VERBDICT_MD5,
'conll05st',
download,
)
self.target_dict_file = target_dict_file
if self.target_dict_file is None:
assert download, (
"target_dict_file is not set and downloading automatically is disabled"
)
self.target_dict_file = _check_exists_and_download(
target_dict_file,
TRGDICT_URL,
TRGDICT_MD5,
'conll05st',
download,
)
self.emb_file = emb_file
if self.emb_file is None:
assert download, (
"emb_file is not set and downloading automatically is disabled"
)
self.emb_file = _check_exists_and_download(
emb_file, EMB_URL, EMB_MD5, 'conll05st', download
)
self.word_dict = self._load_dict(self.word_dict_file)
self.predicate_dict = self._load_dict(self.verb_dict_file)
self.label_dict = self._load_label_dict(self.target_dict_file)
# read dataset into memory
self._load_anno()
def _load_label_dict(self, filename: str) -> dict[str, int]:
d = {}
tag_dict = set()
with open(filename, 'r') as f:
for i, line in enumerate(f):
line = line.strip()
if line.startswith("B-"):
tag_dict.add(line[2:])
elif line.startswith("I-"):
tag_dict.add(line[2:])
index = 0
for tag in tag_dict:
d["B-" + tag] = index
index += 1
d["I-" + tag] = index
index += 1
d["O"] = index
return d
def _load_dict(self, filename: str) -> dict[str, int]:
d = {}
with open(filename, 'r') as f:
for i, line in enumerate(f):
d[line.strip()] = i
return d
def _load_anno(self) -> None:
tf = tarfile.open(self.data_file)
wf = tf.extractfile(
"conll05st-release/test.wsj/words/test.wsj.words.gz"
)
pf = tf.extractfile(
"conll05st-release/test.wsj/props/test.wsj.props.gz"
)
self.sentences = []
self.predicates = []
self.labels = []
with (
gzip.GzipFile(fileobj=wf) as words_file,
gzip.GzipFile(fileobj=pf) as props_file,
):
sentences = []
labels = []
one_seg = []
for word, label in zip(words_file, props_file):
word = word.strip().decode()
label = label.strip().decode().split()
if len(label) == 0: # end of sentence
for i in range(len(one_seg[0])):
a_kind_label = [x[i] for x in one_seg]
labels.append(a_kind_label)
if len(labels) >= 1:
verb_list = []
for x in labels[0]:
if x != '-':
verb_list.append(x)
for i, lbl in enumerate(labels[1:]):
cur_tag = 'O'
is_in_bracket = False
lbl_seq = []
verb_word = ''
for l in lbl:
if l == '*' and not is_in_bracket:
lbl_seq.append('O')
elif l == '*' and is_in_bracket:
lbl_seq.append('I-' + cur_tag)
elif l == '*)':
lbl_seq.append('I-' + cur_tag)
is_in_bracket = False
elif l.find('(') != -1 and l.find(')') != -1:
cur_tag = l[1 : l.find('*')]
lbl_seq.append('B-' + cur_tag)
is_in_bracket = False
elif l.find('(') != -1 and l.find(')') == -1:
cur_tag = l[1 : l.find('*')]
lbl_seq.append('B-' + cur_tag)
is_in_bracket = True
else:
raise RuntimeError(f'Unexpected label: {l}')
self.sentences.append(sentences)
self.predicates.append(verb_list[i])
self.labels.append(lbl_seq)
sentences = []
labels = []
one_seg = []
else:
sentences.append(word)
one_seg.append(label)
pf.close()
wf.close()
tf.close()
def __getitem__(
self, idx: int
) -> tuple[
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
]:
sentence = self.sentences[idx]
predicate = self.predicates[idx]
labels = self.labels[idx]
sen_len = len(sentence)
verb_index = labels.index('B-V')
mark = [0] * len(labels)
if verb_index > 0:
mark[verb_index - 1] = 1
ctx_n1 = sentence[verb_index - 1]
else:
ctx_n1 = 'bos'
if verb_index > 1:
mark[verb_index - 2] = 1
ctx_n2 = sentence[verb_index - 2]
else:
ctx_n2 = 'bos'
mark[verb_index] = 1
ctx_0 = sentence[verb_index]
if verb_index < len(labels) - 1:
mark[verb_index + 1] = 1
ctx_p1 = sentence[verb_index + 1]
else:
ctx_p1 = 'eos'
if verb_index < len(labels) - 2:
mark[verb_index + 2] = 1
ctx_p2 = sentence[verb_index + 2]
else:
ctx_p2 = 'eos'
word_idx = [self.word_dict.get(w, UNK_IDX) for w in sentence]
ctx_n2_idx = [self.word_dict.get(ctx_n2, UNK_IDX)] * sen_len
ctx_n1_idx = [self.word_dict.get(ctx_n1, UNK_IDX)] * sen_len
ctx_0_idx = [self.word_dict.get(ctx_0, UNK_IDX)] * sen_len
ctx_p1_idx = [self.word_dict.get(ctx_p1, UNK_IDX)] * sen_len
ctx_p2_idx = [self.word_dict.get(ctx_p2, UNK_IDX)] * sen_len
pred_idx = [self.predicate_dict.get(predicate)] * sen_len
label_idx = [self.label_dict.get(w) for w in labels]
return (
np.array(word_idx),
np.array(ctx_n2_idx),
np.array(ctx_n1_idx),
np.array(ctx_0_idx),
np.array(ctx_p1_idx),
np.array(ctx_p2_idx),
np.array(pred_idx),
np.array(mark),
np.array(label_idx),
)
def __len__(self) -> int:
return len(self.sentences)
def get_dict(self) -> tuple[dict[str, int], dict[str, int], dict[str, int]]:
"""
Get the word, verb and label dictionary of Wikipedia corpus.
Examples:
.. code-block:: pycon
>>> from paddle.text.datasets import Conll05st
>>> conll05st = Conll05st()
>>> word_dict, predicate_dict, label_dict = conll05st.get_dict()
"""
return self.word_dict, self.predicate_dict, self.label_dict
def get_embedding(self) -> str:
"""
Get the embedding dictionary file.
Examples:
.. code-block:: pycon
>>> from paddle.text.datasets import Conll05st
>>> conll05st = Conll05st()
>>> emb_file = conll05st.get_embedding()
"""
return self.emb_file
python/paddle/text/datasets/imdb.py
+182
View File
@@ -0,0 +1,182 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import collections
import re
import string
import tarfile
from typing import TYPE_CHECKING, Literal
import numpy as np
from paddle.dataset.common import _check_exists_and_download
from paddle.io import Dataset
if TYPE_CHECKING:
from re import Pattern
import numpy.typing as npt
_ImdbDataSetMode = Literal["train", "test"]
__all__ = []
URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz'
MD5 = '7c2ac02c03563afcf9b574c7e56c153a'
class Imdb(Dataset):
"""
Implementation of `IMDB <https://datasets.imdbws.com/>`_ dataset.
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None.
mode(str): 'train' 'test' mode. Default 'train'.
cutoff(int): cutoff number for building word dictionary. Default 150.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True.
Returns:
Dataset: instance of IMDB dataset
Examples:
.. code-block:: pycon
>>> # doctest: +TIMEOUT(75)
>>> import paddle
>>> from paddle.text.datasets import Imdb
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, doc, label):
... return paddle.sum(doc), label
>>> imdb = Imdb(mode='train')
>>> for i in range(10):
... doc, label = imdb[i]
... doc = paddle.to_tensor(doc)
... label = paddle.to_tensor(label)
...
... model = SimpleNet()
... image, label = model(doc, label)
... print(doc.shape, label.shape)
paddle.Size([121]) paddle.Size([1])
paddle.Size([115]) paddle.Size([1])
paddle.Size([386]) paddle.Size([1])
paddle.Size([471]) paddle.Size([1])
paddle.Size([585]) paddle.Size([1])
paddle.Size([206]) paddle.Size([1])
paddle.Size([221]) paddle.Size([1])
paddle.Size([324]) paddle.Size([1])
paddle.Size([166]) paddle.Size([1])
paddle.Size([598]) paddle.Size([1])
"""
data_file: str | None
mode: _ImdbDataSetMode
word_idx: dict[str, int]
docs: list
labels: list
def __init__(
self,
data_file: str | None = None,
mode: _ImdbDataSetMode = 'train',
cutoff: int = 150,
download: bool = True,
) -> None:
assert mode.lower() in [
'train',
'test',
], f"mode should be 'train', 'test', but got {mode}"
self.mode = mode.lower()
self.data_file = data_file
if self.data_file is None:
assert download, (
"data_file is not set and downloading automatically is disabled"
)
self.data_file = _check_exists_and_download(
data_file, URL, MD5, 'imdb', download
)
# Build a word dictionary from the corpus
self.word_idx = self._build_work_dict(cutoff)
# read dataset into memory
self._load_anno()
def _build_work_dict(self, cutoff: int) -> dict[str, int]:
word_freq = collections.defaultdict(int)
pattern = re.compile(r"aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$")
for doc in self._tokenize(pattern):
for word in doc:
word_freq[word] += 1
# Not sure if we should prune less-frequent words here.
word_freq = [x for x in word_freq.items() if x[1] > cutoff]
dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0]))
words, _ = list(zip(*dictionary))
word_idx = dict(list(zip(words, range(len(words)))))
word_idx['<unk>'] = len(words)
return word_idx
def _tokenize(self, pattern: Pattern[str]) -> list[list[str]]:
data = []
with tarfile.open(self.data_file) as tarf:
tf = tarf.next()
while tf is not None:
if bool(pattern.match(tf.name)):
# newline and punctuations removal and ad-hoc tokenization.
data.append(
tarf.extractfile(tf)
.read()
.rstrip(b'\n\r')
.translate(None, string.punctuation.encode('latin-1'))
.lower()
.split()
)
tf = tarf.next()
return data
def _load_anno(self) -> None:
pos_pattern = re.compile(rf"aclImdb/{self.mode}/pos/.*\.txt$")
neg_pattern = re.compile(rf"aclImdb/{self.mode}/neg/.*\.txt$")
UNK = self.word_idx['<unk>']
self.docs = []
self.labels = []
for doc in self._tokenize(pos_pattern):
self.docs.append([self.word_idx.get(w, UNK) for w in doc])
self.labels.append(0)
for doc in self._tokenize(neg_pattern):
self.docs.append([self.word_idx.get(w, UNK) for w in doc])
self.labels.append(1)
def __getitem__(
self, idx: int
) -> tuple[npt.NDArray[np.int_], npt.NDArray[np.int_]]:
return (np.array(self.docs[idx]), np.array([self.labels[idx]]))
def __len__(self) -> int:
return len(self.docs)
python/paddle/text/datasets/imikolov.py
+204
View File
@@ -0,0 +1,204 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import collections
import tarfile
from typing import TYPE_CHECKING, Literal
import numpy as np
from paddle.dataset.common import _check_exists_and_download
from paddle.io import Dataset
if TYPE_CHECKING:
import numpy.typing as npt
_ImikolovDataType = Literal["NGRAM", "SEQ"]
_ImikolovDataSetMode = Literal["train", "test"]
__all__ = []
URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz'
MD5 = '30177ea32e27c525793142b6bf2c8e2d'
class Imikolov(Dataset):
"""
Implementation of imikolov dataset.
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None.
data_type(str): 'NGRAM' or 'SEQ'. Default 'NGRAM'.
window_size(int): sliding window size for 'NGRAM' data. Default -1.
mode(str): 'train' 'test' mode. Default 'train'.
min_word_freq(int): minimal word frequencies for building word dictionary. Default 50.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True
Returns:
Dataset: instance of imikolov dataset
Examples:
.. code-block:: pycon
>>> # doctest: +TIMEOUT(60)
>>> import paddle
>>> from paddle.text.datasets import Imikolov
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, src, trg):
... return paddle.sum(src), paddle.sum(trg)
>>> imikolov = Imikolov(mode='train', data_type='SEQ', window_size=2)
>>> for i in range(10):
... src, trg = imikolov[i]
... src = paddle.to_tensor(src)
... trg = paddle.to_tensor(trg)
...
... model = SimpleNet()
... src, trg = model(src, trg)
... print(src.item(), trg.item())
2076 2075
2076 2075
675 674
4 3
464 463
2076 2075
865 864
2076 2075
2076 2075
1793 1792
"""
data_file: str | None
data_type: _ImikolovDataType
window_size: int
mode: _ImikolovDataSetMode
min_word_freq: int
word_idx: dict[str, int]
def __init__(
self,
data_file: str | None = None,
data_type: _ImikolovDataType = 'NGRAM',
window_size: int = -1,
mode: _ImikolovDataSetMode = 'train',
min_word_freq: int = 50,
download: bool = True,
) -> None:
assert data_type.upper() in [
'NGRAM',
'SEQ',
], f"data type should be 'NGRAM', 'SEQ', but got {data_type}"
self.data_type = data_type.upper()
assert mode.lower() in [
'train',
'test',
], f"mode should be 'train', 'test', but got {mode}"
self.mode = mode.lower()
self.window_size = window_size
self.min_word_freq = min_word_freq
self.data_file = data_file
if self.data_file is None:
assert download, (
"data_file is not set and downloading automatically disabled"
)
self.data_file = _check_exists_and_download(
data_file, URL, MD5, 'imikolov', download
)
# Build a word dictionary from the corpus
self.word_idx = self._build_work_dict(min_word_freq)
# read dataset into memory
self._load_anno()
def word_count(self, f, word_freq=None):
if word_freq is None:
word_freq = collections.defaultdict(int)
for l in f:
for w in l.strip().split():
word_freq[w] += 1
word_freq['<s>'] += 1
word_freq['<e>'] += 1
return word_freq
def _build_work_dict(self, cutoff: int) -> dict[str, int]:
train_filename = './simple-examples/data/ptb.train.txt'
test_filename = './simple-examples/data/ptb.valid.txt'
with tarfile.open(self.data_file) as tf:
trainf = tf.extractfile(train_filename)
testf = tf.extractfile(test_filename)
word_freq = self.word_count(testf, self.word_count(trainf))
if '<unk>' in word_freq:
# remove <unk> for now, since we will set it as last index
del word_freq['<unk>']
word_freq = [
x for x in word_freq.items() if x[1] > self.min_word_freq
]
word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0]))
words, _ = list(zip(*word_freq_sorted))
word_idx = dict(list(zip(words, range(len(words)))))
word_idx['<unk>'] = len(words)
return word_idx
def _load_anno(self) -> None:
self.data = []
with tarfile.open(self.data_file) as tf:
filename = f'./simple-examples/data/ptb.{self.mode}.txt'
f = tf.extractfile(filename)
UNK = self.word_idx['<unk>']
for l in f:
if self.data_type == 'NGRAM':
assert self.window_size > -1, 'Invalid gram length'
l = ["<s>", *l.strip().split(), "<e>"]
if len(l) >= self.window_size:
l = [self.word_idx.get(w, UNK) for w in l]
for i in range(self.window_size, len(l) + 1):
self.data.append(tuple(l[i - self.window_size : i]))
elif self.data_type == 'SEQ':
l = l.strip().split()
l = [self.word_idx.get(w, UNK) for w in l]
src_seq = [self.word_idx["<s>"], *l]
trg_seq = [*l, self.word_idx["<e>"]]
if self.window_size > 0 and len(src_seq) > self.window_size:
continue
self.data.append((src_seq, trg_seq))
else:
raise AssertionError('Unknown data type')
def __getitem__(
self, idx: int
) -> tuple[npt.NDArray[np.int_], npt.NDArray[np.int_]]:
return tuple([np.array(d) for d in self.data[idx]])
def __len__(self) -> int:
return len(self.data)
python/paddle/text/datasets/movielens.py
+265
View File
@@ -0,0 +1,265 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import re
import zipfile
from typing import TYPE_CHECKING, Any, Literal
import numpy as np
from paddle.dataset.common import _check_exists_and_download
from paddle.io import Dataset
if TYPE_CHECKING:
import numpy.typing as npt
_MovieLensDataSetMode = Literal["train", "test"]
__all__ = []
age_table = [1, 18, 25, 35, 45, 50, 56]
URL = 'https://dataset.bj.bcebos.com/movielens%2Fml-1m.zip'
MD5 = 'c4d9eecfca2ab87c1945afe126590906'
class MovieInfo:
"""
Movie id, title and categories information are stored in MovieInfo.
"""
index: int
categories: list[str]
title: str
def __init__(self, index: str, categories: list[str], title: str) -> None:
self.index = int(index)
self.categories = categories
self.title = title
def value(self, categories_dict, movie_title_dict):
"""
Get information from a movie.
"""
return [
[self.index],
[categories_dict[c] for c in self.categories],
[movie_title_dict[w.lower()] for w in self.title.split()],
]
def __str__(self) -> str:
return f"<MovieInfo id({self.index}), title({self.title}), categories({self.categories})>"
def __repr__(self) -> str:
return self.__str__()
class UserInfo:
"""
User id, gender, age, and job information are stored in UserInfo.
"""
index: int
is_male: bool
age: int
job_id: int
def __init__(self, index: str, gender: str, age: str, job_id: str) -> None:
self.index = int(index)
self.is_male = gender == 'M'
self.age = age_table.index(int(age))
self.job_id = int(job_id)
def value(self):
"""
Get information from a user.
"""
return [
[self.index],
[0 if self.is_male else 1],
[self.age],
[self.job_id],
]
def __str__(self) -> str:
gender = "M" if self.is_male else "F"
return f"<UserInfo id({self.index}), gender({gender}), age({age_table[self.age]}), job({self.job_id})>"
def __repr__(self) -> str:
return str(self)
class Movielens(Dataset):
"""
Implementation of `Movielens 1-M <https://grouplens.org/datasets/movielens/1m/>`_ dataset.
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None.
mode(str): 'train' or 'test' mode. Default 'train'.
test_ratio(float): split ratio for test sample. Default 0.1.
rand_seed(int): random seed. Default 0.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True.
Returns:
Dataset: instance of Movielens 1-M dataset.
Examples:
.. code-block:: pycon
>>> # doctest: +TIMEOUT(75)
>>> import paddle
>>> from paddle.text.datasets import Movielens
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, category, title, rating):
... return paddle.sum(category), paddle.sum(title), paddle.sum(rating)
>>> movielens = Movielens(mode='train')
>>> for i in range(10):
... category, title, rating = movielens[i][-3:]
... category = paddle.to_tensor(category)
... title = paddle.to_tensor(title)
... rating = paddle.to_tensor(rating)
...
... model = SimpleNet()
... category, title, rating = model(category, title, rating)
... print(category.shape, title.shape, rating.shape)
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
paddle.Size([]) paddle.Size([]) paddle.Size([])
"""
mode: _MovieLensDataSetMode
data_file: str | None
test_ratio: float
rand_seed: int
movie_info: dict[int, MovieInfo]
movie_title_dict: dict[str, int]
categories_dict: dict[str, int]
user_info: dict[int, UserInfo]
data: list[list[float]]
def __init__(
self,
data_file: str | None = None,
mode: _MovieLensDataSetMode = 'train',
test_ratio: float = 0.1,
rand_seed: int = 0,
download: bool = True,
) -> None:
assert mode.lower() in [
'train',
'test',
], f"mode should be 'train', 'test', but got {mode}"
self.mode = mode.lower()
self.data_file = data_file
if self.data_file is None:
assert download, (
"data_file is not set and downloading automatically is disabled"
)
self.data_file = _check_exists_and_download(
data_file, URL, MD5, 'sentiment', download
)
self.test_ratio = test_ratio
self.rand_seed = rand_seed
np.random.seed(rand_seed)
self._load_meta_info()
self._load_data()
def _load_meta_info(self) -> None:
pattern = re.compile(r'^(.*)\((\d+)\)$')
self.movie_info = {}
self.movie_title_dict = {}
self.categories_dict = {}
self.user_info = {}
with zipfile.ZipFile(self.data_file) as package:
for info in package.infolist():
assert isinstance(info, zipfile.ZipInfo)
title_word_set = set()
categories_set = set()
with package.open('ml-1m/movies.dat') as movie_file:
for i, line in enumerate(movie_file):
line = line.decode(encoding='latin')
movie_id, title, categories = line.strip().split('::')
categories = categories.split('|')
for c in categories:
categories_set.add(c)
title = pattern.match(title).group(1)
self.movie_info[int(movie_id)] = MovieInfo(
index=movie_id, categories=categories, title=title
)
for w in title.split():
title_word_set.add(w.lower())
for i, w in enumerate(title_word_set):
self.movie_title_dict[w] = i
for i, c in enumerate(categories_set):
self.categories_dict[c] = i
with package.open('ml-1m/users.dat') as user_file:
for line in user_file:
line = line.decode(encoding='latin')
uid, gender, age, job, _ = line.strip().split("::")
self.user_info[int(uid)] = UserInfo(
index=uid, gender=gender, age=age, job_id=job
)
def _load_data(self) -> None:
self.data = []
is_test = self.mode == 'test'
with (
zipfile.ZipFile(self.data_file) as package,
package.open('ml-1m/ratings.dat') as rating,
):
for line in rating:
line = line.decode(encoding='latin')
if (np.random.random() < self.test_ratio) == is_test:
uid, mov_id, rating, _ = line.strip().split("::")
uid = int(uid)
mov_id = int(mov_id)
rating = float(rating) * 2 - 5.0
mov = self.movie_info[mov_id]
usr = self.user_info[uid]
self.data.append(
usr.value()
+ mov.value(self.categories_dict, self.movie_title_dict)
+ [[rating]]
)
def __getitem__(self, idx: int) -> tuple[npt.NDArray[Any], ...]:
data = self.data[idx]
return tuple([np.array(d) for d in data])
def __len__(self) -> int:
return len(self.data)
python/paddle/text/datasets/uci_housing.py
+160
View File
@@ -0,0 +1,160 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING, Any, Literal
import numpy as np
import paddle
from paddle.dataset.common import _check_exists_and_download
from paddle.io import Dataset
if TYPE_CHECKING:
import numpy.typing as npt
from paddle._typing.dtype_like import _DTypeLiteral
_UciHousingDataSetMode = Literal["train", "test"]
__all__ = []
URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data'
MD5 = 'd4accdce7a25600298819f8e28e8d593'
feature_names = [
'CRIM',
'ZN',
'INDUS',
'CHAS',
'NOX',
'RM',
'AGE',
'DIS',
'RAD',
'TAX',
'PTRATIO',
'B',
'LSTAT',
]
class UCIHousing(Dataset):
"""
Implementation of `UCI housing <https://archive.ics.uci.edu/ml/datasets/Housing>`_
dataset
Args:
data_file(str|None): path to data file, can be set None if
:attr:`download` is True. Default None.
mode(str): 'train' or 'test' mode. Default 'train'.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True.
Returns:
Dataset: instance of UCI housing dataset.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.text.datasets import UCIHousing
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, feature, target):
... return paddle.sum(feature), target
>>> paddle.disable_static()
>>> uci_housing = UCIHousing(mode='train')
>>> for i in range(10):
... feature, target = uci_housing[i]
... feature = paddle.to_tensor(feature)
... target = paddle.to_tensor(target)
...
... model = SimpleNet()
... feature, target = model(feature, target)
... print(feature.shape, target.numpy())
paddle.Size([]) [24.]
paddle.Size([]) [21.6]
paddle.Size([]) [34.7]
paddle.Size([]) [33.4]
paddle.Size([]) [36.2]
paddle.Size([]) [28.7]
paddle.Size([]) [22.9]
paddle.Size([]) [27.1]
paddle.Size([]) [16.5]
paddle.Size([]) [18.9]
"""
mode: _UciHousingDataSetMode
data_file: str | None
dtype: _DTypeLiteral
def __init__(
self,
data_file: str | None = None,
mode: _UciHousingDataSetMode = 'train',
download: bool = True,
) -> None:
assert mode.lower() in [
'train',
'test',
], f"mode should be 'train' or 'test', but got {mode}"
self.mode = mode.lower()
self.data_file = data_file
if self.data_file is None:
assert download, (
"data_file is not set and downloading automatically is disabled"
)
self.data_file = _check_exists_and_download(
data_file, URL, MD5, 'uci_housing', download
)
# read dataset into memory
self._load_data()
self.dtype = paddle.get_default_dtype()
def _load_data(self, feature_num: int = 14, ratio: float = 0.8) -> None:
data = np.fromfile(self.data_file, sep=' ')
data = data.reshape(data.shape[0] // feature_num, feature_num)
maximums, minimums, avgs = (
data.max(axis=0),
data.min(axis=0),
data.sum(axis=0) / data.shape[0],
)
for i in range(feature_num - 1):
data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
offset = int(data.shape[0] * ratio)
if self.mode == 'train':
self.data = data[:offset]
elif self.mode == 'test':
self.data = data[offset:]
def __getitem__(
self, idx: int
) -> tuple[npt.NDArray[Any], npt.NDArray[Any]]:
data = self.data[idx]
return np.array(data[:-1]).astype(self.dtype), np.array(
data[-1:]
).astype(self.dtype)
def __len__(self) -> int:
return len(self.data)
python/paddle/text/datasets/wmt14.py
+260
View File
@@ -0,0 +1,260 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import tarfile
from typing import TYPE_CHECKING, Literal, overload
import numpy as np
from paddle.dataset.common import _check_exists_and_download
from paddle.io import Dataset
if TYPE_CHECKING:
import numpy.typing as npt
_Wmt14DataSetMode = Literal["train", "test", "gen"]
__all__ = []
URL_DEV_TEST = (
'http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz'
)
MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5'
# this is a small set of data for test. The original data is too large and
# will be add later.
URL_TRAIN = 'http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz'
MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c'
START = "<s>"
END = "<e>"
UNK = "<unk>"
UNK_IDX = 2
class WMT14(Dataset):
"""
Implementation of `WMT14 <http://www.statmt.org/wmt14/>`_ test dataset.
The original WMT14 dataset is too large and a small set of data for set is
provided. This module will download dataset from
http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz .
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None.
mode(str): 'train', 'test' or 'gen'. Default 'train'.
dict_size(int): word dictionary size. Default -1.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True.
Returns:
Dataset: Instance of WMT14 dataset
- src_ids (np.array) - The sequence of token ids of source language.
- trg_ids (np.array) - The sequence of token ids of target language.
- trg_ids_next (np.array) - The next sequence of token ids of target language.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.text.datasets import WMT14
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, src_ids, trg_ids, trg_ids_next):
... return paddle.sum(src_ids), paddle.sum(trg_ids), paddle.sum(trg_ids_next)
>>> wmt14 = WMT14(mode='train', dict_size=50)
>>> for i in range(10):
... src_ids, trg_ids, trg_ids_next = wmt14[i]
... src_ids = paddle.to_tensor(src_ids)
... trg_ids = paddle.to_tensor(trg_ids)
... trg_ids_next = paddle.to_tensor(trg_ids_next)
...
... model = SimpleNet()
... src_ids, trg_ids, trg_ids_next = model(src_ids, trg_ids, trg_ids_next)
... print(src_ids.item(), trg_ids.item(), trg_ids_next.item())
91 38 39
123 81 82
556 229 230
182 26 27
447 242 243
116 110 111
403 288 289
258 221 222
136 34 35
281 136 137
"""
mode: _Wmt14DataSetMode
data_file: str | None
dict_size: int
src_ids: list[list[int]]
trg_ids: list[list[int]]
trg_ids_next: list[list[int]]
src_dict: dict[str, int]
trg_dict: dict[str, int]
def __init__(
self,
data_file: str | None = None,
mode: _Wmt14DataSetMode = 'train',
dict_size: int = -1,
download: bool = True,
) -> None:
assert mode.lower() in [
'train',
'test',
'gen',
], f"mode should be 'train', 'test' or 'gen', but got {mode}"
self.mode = mode.lower()
self.data_file = data_file
if self.data_file is None:
assert download, (
"data_file is not set and downloading automatically is disabled"
)
self.data_file = _check_exists_and_download(
data_file, URL_TRAIN, MD5_TRAIN, 'wmt14', download
)
# read dataset into memory
assert dict_size > 0, "dict_size should be set as positive number"
self.dict_size = dict_size
self._load_data()
def _load_data(self) -> None:
def __to_dict(fd, size: int) -> dict[str, int]:
out_dict = {}
for line_count, line in enumerate(fd):
if line_count < size:
out_dict[line.strip().decode()] = line_count
else:
break
return out_dict
self.src_ids = []
self.trg_ids = []
self.trg_ids_next = []
with tarfile.open(self.data_file, mode='r') as f:
names = [
each_item.name
for each_item in f
if each_item.name.endswith("src.dict")
]
assert len(names) == 1
self.src_dict = __to_dict(f.extractfile(names[0]), self.dict_size)
names = [
each_item.name
for each_item in f
if each_item.name.endswith("trg.dict")
]
assert len(names) == 1
self.trg_dict = __to_dict(f.extractfile(names[0]), self.dict_size)
file_name = f"{self.mode}/{self.mode}"
names = [
each_item.name
for each_item in f
if each_item.name.endswith(file_name)
]
for name in names:
for line in f.extractfile(name):
line = line.decode()
line_split = line.strip().split('\t')
if len(line_split) != 2:
continue
src_seq = line_split[0] # one source sequence
src_words = src_seq.split()
src_ids = [
self.src_dict.get(w, UNK_IDX)
for w in [START, *src_words, END]
]
trg_seq = line_split[1] # one target sequence
trg_words = trg_seq.split()
trg_ids = [self.trg_dict.get(w, UNK_IDX) for w in trg_words]
# remove sequence whose length > 80 in training mode
if len(src_ids) > 80 or len(trg_ids) > 80:
continue
trg_ids_next = [*trg_ids, self.trg_dict[END]]
trg_ids = [self.trg_dict[START], *trg_ids]
self.src_ids.append(src_ids)
self.trg_ids.append(trg_ids)
self.trg_ids_next.append(trg_ids_next)
def __getitem__(
self, idx: int
) -> tuple[
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
]:
return (
np.array(self.src_ids[idx]),
np.array(self.trg_ids[idx]),
np.array(self.trg_ids_next[idx]),
)
def __len__(self) -> int:
return len(self.src_ids)
@overload
def get_dict(
self, reverse: Literal[True] = ...
) -> tuple[dict[int, str], dict[int, str]]: ...
@overload
def get_dict(
self, reverse: Literal[False] = ...
) -> tuple[dict[str, int], dict[str, int]]: ...
@overload
def get_dict(
self, reverse: bool = ...
) -> (
tuple[dict[str, int], dict[str, int]]
| tuple[dict[int, str], dict[int, str]]
): ...
def get_dict(self, reverse=False):
"""
Get the source and target dictionary.
Args:
reverse (bool): whether to reverse key and value in dictionary,
i.e. key: value to value: key.
Returns:
Two dictionaries, the source and target dictionary.
Examples:
.. code-block:: pycon
>>> from paddle.text.datasets import WMT14
>>> wmt14 = WMT14(mode='train', dict_size=50)
>>> src_dict, trg_dict = wmt14.get_dict()
"""
src_dict, trg_dict = self.src_dict, self.trg_dict
if reverse:
src_dict = {v: k for k, v in src_dict.items()}
trg_dict = {v: k for k, v in trg_dict.items()}
return src_dict, trg_dict
python/paddle/text/datasets/wmt16.py
+341
View File
@@ -0,0 +1,341 @@
# Copyright (c) 2020 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
import os
import tarfile
from collections import defaultdict
from typing import TYPE_CHECKING, Literal, overload
import numpy as np
import paddle
from paddle.dataset.common import _check_exists_and_download
from paddle.io import Dataset
if TYPE_CHECKING:
import numpy.typing as npt
_Wmt16DataSetMode = Literal["train", "test", "val"]
_Wmt16Language = Literal["en", "de"]
__all__ = []
DATA_URL = "http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz"
DATA_MD5 = "0c38be43600334966403524a40dcd81e"
TOTAL_EN_WORDS = 11250
TOTAL_DE_WORDS = 19220
START_MARK = "<s>"
END_MARK = "<e>"
UNK_MARK = "<unk>"
class WMT16(Dataset):
"""
Implementation of `WMT16 <http://www.statmt.org/wmt16/>`_ test dataset.
ACL2016 Multimodal Machine Translation. Please see this website for more
details: http://www.statmt.org/wmt16/multimodal-task.html#task1
If you use the dataset created for your task, please cite the following paper:
Multi30K: Multilingual English-German Image Descriptions.
.. code-block:: text
@article{elliott-EtAl:2016:VL16,
author = {{Elliott}, D. and {Frank}, S. and {Sima"an}, K. and {Specia}, L.},
title = {Multi30K: Multilingual English-German Image Descriptions},
booktitle = {Proceedings of the 6th Workshop on Vision and Language},
year = {2016},
pages = {70--74},
year = 2016
}
Args:
data_file(str|None): path to data tar file, can be set None if
:attr:`download` is True. Default None.
mode(str): 'train', 'test' or 'val'. Default 'train'.
src_dict_size(int): word dictionary size for source language word. Default -1.
trg_dict_size(int): word dictionary size for target language word. Default -1.
lang(str): source language, 'en' or 'de'. Default 'en'.
download(bool): whether to download dataset automatically if
:attr:`data_file` is not set. Default True.
Returns:
Dataset: Instance of WMT16 dataset. The instance of dataset has 3 fields:
- src_ids (np.array) - The sequence of token ids of source language.
- trg_ids (np.array) - The sequence of token ids of target language.
- trg_ids_next (np.array) - The next sequence of token ids of target language.
Examples:
.. code-block:: pycon
>>> import paddle
>>> from paddle.text.datasets import WMT16
>>> class SimpleNet(paddle.nn.Layer):
... def __init__(self):
... super().__init__()
...
... def forward(self, src_ids, trg_ids, trg_ids_next):
... return paddle.sum(src_ids), paddle.sum(trg_ids), paddle.sum(trg_ids_next)
>>> wmt16 = WMT16(mode='train', src_dict_size=50, trg_dict_size=50)
>>> for i in range(10):
... src_ids, trg_ids, trg_ids_next = wmt16[i]
... src_ids = paddle.to_tensor(src_ids)
... trg_ids = paddle.to_tensor(trg_ids)
... trg_ids_next = paddle.to_tensor(trg_ids_next)
...
... model = SimpleNet()
... src_ids, trg_ids, trg_ids_next = model(src_ids, trg_ids, trg_ids_next)
... print(src_ids.item(), trg_ids.item(), trg_ids_next.item())
89 32 33
79 18 19
55 26 27
147 36 37
106 22 23
135 50 51
54 43 44
217 30 31
146 51 52
55 24 25
"""
mode: _Wmt16DataSetMode
data_file: str | None
lang: _Wmt16Language
src_dict_size: int
trg_dict_size: int
src_dict: dict[str, int]
trg_dict: dict[str, int]
src_ids: list[list[int]]
trg_ids: list[list[int]]
trg_ids_next: list[list[int]]
def __init__(
self,
data_file: str | None = None,
mode: _Wmt16DataSetMode = 'train',
src_dict_size: int = -1,
trg_dict_size: int = -1,
lang: _Wmt16Language = 'en',
download: bool = True,
) -> None:
assert mode.lower() in [
'train',
'test',
'val',
], f"mode should be 'train', 'test' or 'val', but got {mode}"
self.mode = mode.lower()
self.data_file = data_file
if self.data_file is None:
assert download, (
"data_file is not set and downloading automatically is disabled"
)
self.data_file = _check_exists_and_download(
data_file, DATA_URL, DATA_MD5, 'wmt16', download
)
self.lang = lang
assert src_dict_size > 0, "dict_size should be set as positive number"
assert trg_dict_size > 0, "dict_size should be set as positive number"
self.src_dict_size = min(
src_dict_size, (TOTAL_EN_WORDS if lang == "en" else TOTAL_DE_WORDS)
)
self.trg_dict_size = min(
trg_dict_size, (TOTAL_DE_WORDS if lang == "en" else TOTAL_EN_WORDS)
)
# load source and target word dict
self.src_dict = self._load_dict(lang, src_dict_size)
self.trg_dict = self._load_dict(
"de" if lang == "en" else "en", trg_dict_size
)
# load data
self.data = self._load_data()
@overload
def _load_dict(
self, lang: _Wmt16Language, dict_size: int, reverse: Literal[True] = ...
) -> dict[int, str]: ...
@overload
def _load_dict(
self,
lang: _Wmt16Language,
dict_size: int,
reverse: Literal[False] = ...,
) -> dict[str, int]: ...
@overload
def _load_dict(
self, lang: _Wmt16Language, dict_size: int, reverse: bool = ...
) -> dict[int, str] | dict[str, int]: ...
def _load_dict(self, lang, dict_size, reverse=False):
dict_path = os.path.join(
paddle.dataset.common.DATA_HOME,
f"wmt16/{lang}_{dict_size}.dict",
)
dict_found = False
if os.path.exists(dict_path):
with open(dict_path, "rb") as d:
dict_found = len(d.readlines()) == dict_size
if not dict_found:
self._build_dict(dict_path, dict_size, lang)
word_dict = {}
with open(dict_path, "rb") as fdict:
for idx, line in enumerate(fdict):
if reverse:
word_dict[idx] = line.strip().decode()
else:
word_dict[line.strip().decode()] = idx
return word_dict
def _build_dict(
self, dict_path: str, dict_size: int, lang: _Wmt16Language
) -> None:
word_dict = defaultdict(int)
with tarfile.open(self.data_file, mode="r") as f:
for line in f.extractfile("wmt16/train"):
line = line.decode()
line_split = line.strip().split("\t")
if len(line_split) != 2:
continue
sen = line_split[0] if self.lang == "en" else line_split[1]
for w in sen.split():
word_dict[w] += 1
with open(dict_path, "wb") as fout:
fout.write((f"{START_MARK}\n{END_MARK}\n{UNK_MARK}\n").encode())
for idx, word in enumerate(
sorted(word_dict.items(), key=lambda x: x[1], reverse=True)
):
if idx + 3 == dict_size:
break
fout.write(word[0].encode())
fout.write(b'\n')
def _load_data(self) -> None:
# the index for start mark, end mark, and unk are the same in source
# language and target language. Here uses the source language
# dictionary to determine their indices.
start_id = self.src_dict[START_MARK]
end_id = self.src_dict[END_MARK]
unk_id = self.src_dict[UNK_MARK]
src_col = 0 if self.lang == "en" else 1
trg_col = 1 - src_col
self.src_ids = []
self.trg_ids = []
self.trg_ids_next = []
with tarfile.open(self.data_file, mode="r") as f:
for line in f.extractfile(f"wmt16/{self.mode}"):
line = line.decode()
line_split = line.strip().split("\t")
if len(line_split) != 2:
continue
src_words = line_split[src_col].split()
src_ids = (
[start_id]
+ [self.src_dict.get(w, unk_id) for w in src_words]
+ [end_id]
)
trg_words = line_split[trg_col].split()
trg_ids = [self.trg_dict.get(w, unk_id) for w in trg_words]
trg_ids_next = [*trg_ids, end_id]
trg_ids = [start_id, *trg_ids]
self.src_ids.append(src_ids)
self.trg_ids.append(trg_ids)
self.trg_ids_next.append(trg_ids_next)
def __getitem__(
self, idx: int
) -> tuple[
npt.NDArray[np.int_],
npt.NDArray[np.int_],
npt.NDArray[np.int_],
]:
return (
np.array(self.src_ids[idx]),
np.array(self.trg_ids[idx]),
np.array(self.trg_ids_next[idx]),
)
def __len__(self) -> int:
return len(self.src_ids)
@overload
def get_dict(
self, lang: _Wmt16Language, reverse: Literal[True] = ...
) -> dict[int, str]: ...
@overload
def get_dict(
self, lang: _Wmt16Language, reverse: Literal[False] = ...
) -> dict[str, int]: ...
@overload
def get_dict(
self, lang: _Wmt16Language, reverse: bool = ...
) -> dict[int, str] | dict[str, int]: ...
def get_dict(self, lang, reverse=False):
"""
return the word dictionary for the specified language.
Args:
lang(string): A string indicating which language is the source
language. Available options are: "en" for English
and "de" for Germany.
reverse(bool): If reverse is set to False, the returned python
dictionary will use word as key and use index as value.
If reverse is set to True, the returned python
dictionary will use index as key and word as value.
Returns:
dict: The word dictionary for the specific language.
Examples:
.. code-block:: pycon
>>> from paddle.text.datasets import WMT16
>>> wmt16 = WMT16(mode='train', src_dict_size=50, trg_dict_size=50)
>>> en_dict = wmt16.get_dict('en')
"""
dict_size = (
self.src_dict_size if lang == self.lang else self.trg_dict_size
)
dict_path = os.path.join(
paddle.dataset.common.DATA_HOME,
f"wmt16/{lang}_{dict_size}.dict",
)
assert os.path.exists(dict_path), "Word dictionary does not exist. "
"Please invoke paddle.dataset.wmt16.train/test/validation first "
"to build the dictionary."
return self._load_dict(lang, dict_size)
python/paddle/text/viterbi_decode.py
+176
View File
@@ -0,0 +1,176 @@
# Copyright (c) 2021 PaddlePaddle Authors. All Rights Reserved.
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
from __future__ import annotations
from typing import TYPE_CHECKING
from paddle import _C_ops
from ..base.data_feeder import check_type, check_variable_and_dtype
from ..base.framework import in_dynamic_or_pir_mode
from ..base.layer_helper import LayerHelper
from ..nn import Layer
if TYPE_CHECKING:
from paddle import Tensor
__all__ = ['viterbi_decode', 'ViterbiDecoder']
def viterbi_decode(
potentials: Tensor,
transition_params: Tensor,
lengths: Tensor,
include_bos_eos_tag: bool = True,
name: str | None = None,
) -> Tensor:
"""
Decode the highest scoring sequence of tags computed by transitions and potentials and get the viterbi path.
Args:
potentials (Tensor): The input tensor of unary emission. This is a 3-D
tensor with shape of [batch_size, sequence_length, num_tags]. The data type is float32 or float64.
transition_params (Tensor): The input tensor of transition matrix. This is a 2-D
tensor with shape of [num_tags, num_tags]. The data type is float32 or float64.
lengths (Tensor): The input tensor of length of each sequence. This is a 1-D tensor with shape of [batch_size]. The data type is int64.
include_bos_eos_tag (`bool`, optional): If set to True, the last row and the last column of transitions will be considered
as start tag, the second to last row and the second to last column of transitions will be considered as stop tag. Defaults to ``True``.
name (str|None, optional): The default value is None. Normally there is no need for user to set this property. For more information, please
refer to :ref:`api_guide_Name`.
Returns:
scores(Tensor): The output tensor containing the score for the Viterbi sequence. The shape is [batch_size]
and the data type is float32 or float64.
paths(Tensor): The output tensor containing the highest scoring tag indices. The shape is [batch_size, sequence_length]
and the data type is int64.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(2023)
>>> batch_size, seq_len, num_tags = 2, 4, 3
>>> emission = paddle.rand((batch_size, seq_len, num_tags), dtype='float32')
>>> length = paddle.randint(1, seq_len + 1, [batch_size])
>>> tags = paddle.randint(0, num_tags, [batch_size, seq_len])
>>> transition = paddle.rand((num_tags, num_tags), dtype='float32')
>>> scores, path = paddle.text.viterbi_decode(emission, transition, length, False)
>>> print(scores)
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
[2.57385254, 2.04533720])
>>> print(path)
Tensor(shape=[2, 2], dtype=int64, place=Place(cpu), stop_gradient=True,
[[0, 0],
[1, 1]])
"""
if in_dynamic_or_pir_mode():
return _C_ops.viterbi_decode(
potentials, transition_params, lengths, include_bos_eos_tag
)
check_variable_and_dtype(
potentials, 'input', ['float32', 'float64'], 'viterbi_decode'
)
check_variable_and_dtype(
transition_params,
'transitions',
['float32', 'float64'],
'viterbi_decode',
)
check_variable_and_dtype(lengths, 'length', 'int64', 'viterbi_decode')
check_type(include_bos_eos_tag, 'include_tag', bool, 'viterbi_decode')
helper = LayerHelper('viterbi_decode', **locals())
attrs = {'include_bos_eos_tag': include_bos_eos_tag}
scores = helper.create_variable_for_type_inference(potentials.dtype)
path = helper.create_variable_for_type_inference('int64')
helper.append_op(
type='viterbi_decode',
inputs={
'Input': potentials,
'Transition': transition_params,
'Length': lengths,
},
outputs={'Scores': scores, 'Path': path},
attrs=attrs,
)
return scores, path
class ViterbiDecoder(Layer):
"""
Decode the highest scoring sequence of tags computed by transitions and potentials and get the viterbi path.
Args:
transitions (`Tensor`): The transition matrix. Its dtype is float32 and has a shape of `[num_tags, num_tags]`.
include_bos_eos_tag (`bool`, optional): If set to True, the last row and the last column of transitions will be considered
as start tag, the second to last row and the second to last column of transitions will be considered as stop tag. Defaults to ``True``.
name (str|None, optional): The default value is None. Normally there is no need for user to set this property. For more information, please
refer to :ref:`api_guide_Name`.
Shape:
potentials (Tensor): The input tensor of unary emission. This is a 3-D tensor with shape of
[batch_size, sequence_length, num_tags]. The data type is float32 or float64.
lengths (Tensor): The input tensor of length of each sequence. This is a 1-D tensor with shape of
[batch_size]. The data type is int64.
Returns:
scores(Tensor): The output tensor containing the score for the Viterbi sequence. The shape is [batch_size]
and the data type is float32 or float64.
paths(Tensor): The output tensor containing the highest scoring tag indices. The shape is [batch_size, sequence_length]
and the data type is int64.
Examples:
.. code-block:: pycon
>>> import paddle
>>> paddle.seed(2023)
>>> batch_size, seq_len, num_tags = 2, 4, 3
>>> emission = paddle.rand((batch_size, seq_len, num_tags), dtype='float32')
>>> length = paddle.randint(1, seq_len + 1, [batch_size])
>>> tags = paddle.randint(0, num_tags, [batch_size, seq_len])
>>> transition = paddle.rand((num_tags, num_tags), dtype='float32')
>>> decoder = paddle.text.ViterbiDecoder(transition, include_bos_eos_tag=False)
>>> scores, path = decoder(emission, length)
>>> print(scores)
Tensor(shape=[2], dtype=float32, place=Place(cpu), stop_gradient=True,
[2.57385254, 2.04533720])
>>> print(path)
Tensor(shape=[2, 2], dtype=int64, place=Place(cpu), stop_gradient=True,
[[0, 0],
[1, 1]])
"""
transitions: Tensor
include_bos_eos_tag: bool
name: str | None
def __init__(
self,
transitions: Tensor,
include_bos_eos_tag: bool = True,
name: str | None = None,
) -> None:
super().__init__()
self.transitions = transitions
self.include_bos_eos_tag = include_bos_eos_tag
self.name = name
def forward(self, potentials: Tensor, lengths: Tensor) -> Tensor:
return viterbi_decode(
potentials,
self.transitions,
lengths,
self.include_bos_eos_tag,
self.name,
)