chore: import upstream snapshot with attribution
This commit is contained in:
wehub-resource-sync
@@ -0,0 +1,34 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Dataset package.
|
||||
"""
|
||||
|
||||
from . import ( # noqa: F401
|
||||
cifar,
|
||||
conll05,
|
||||
flowers,
|
||||
image,
|
||||
imdb,
|
||||
imikolov,
|
||||
mnist,
|
||||
movielens,
|
||||
uci_housing,
|
||||
voc2012,
|
||||
wmt14,
|
||||
wmt16,
|
||||
)
|
||||
|
||||
# set __all__ as empty for not showing APIs under paddle.dataset
|
||||
__all__ = []
|
||||
@@ -0,0 +1,195 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
CIFAR dataset.
|
||||
|
||||
This module will download dataset from https://dataset.bj.bcebos.com/cifar/cifar-10-python.tar.gz and https://dataset.bj.bcebos.com/cifar/cifar-100-python.tar.gz, parse train/test set into
|
||||
paddle reader creators.
|
||||
|
||||
The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes,
|
||||
with 6000 images per class. There are 50000 training images and 10000 test
|
||||
images.
|
||||
|
||||
The CIFAR-100 dataset is just like the CIFAR-10, except it has 100 classes
|
||||
containing 600 images each. There are 500 training images and 100 testing
|
||||
images per class.
|
||||
|
||||
"""
|
||||
|
||||
import os
|
||||
import pickle
|
||||
import tarfile
|
||||
|
||||
import numpy
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
URL_PREFIX = 'https://dataset.bj.bcebos.com/cifar/'
|
||||
CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz'
|
||||
CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a'
|
||||
CIFAR100_URL = URL_PREFIX + 'cifar-100-python.tar.gz'
|
||||
CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85'
|
||||
|
||||
|
||||
def reader_creator(filename, sub_name, cycle=False, md5sum=None):
|
||||
def read_batch(batch):
|
||||
data = batch[b'data']
|
||||
labels = batch.get(b'labels', batch.get(b'fine_labels', None))
|
||||
assert labels is not None
|
||||
for sample, label in zip(data, labels):
|
||||
yield (sample / 255.0).astype(numpy.float32), int(label)
|
||||
|
||||
verified_stat = None
|
||||
|
||||
def reader():
|
||||
nonlocal verified_stat
|
||||
if md5sum is not None:
|
||||
stat = os.stat(filename)
|
||||
current_stat = (stat.st_mtime_ns, stat.st_size)
|
||||
if verified_stat != current_stat:
|
||||
file_md5 = paddle.dataset.common.md5file(filename)
|
||||
if file_md5 != md5sum:
|
||||
raise ValueError(
|
||||
"Loading unverified CIFAR pickle archive disabled. "
|
||||
f"Please use the official MD5 {md5sum}."
|
||||
)
|
||||
verified_stat = current_stat
|
||||
while True:
|
||||
with tarfile.open(filename, mode='r') as f:
|
||||
names = (
|
||||
each_item.name
|
||||
for each_item in f
|
||||
if sub_name in each_item.name
|
||||
)
|
||||
|
||||
for name in names:
|
||||
batch = pickle.load(f.extractfile(name), encoding='bytes')
|
||||
yield from read_batch(batch)
|
||||
|
||||
if not cycle:
|
||||
break
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Cifar100",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train100():
|
||||
"""
|
||||
CIFAR-100 training set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is image pixels in
|
||||
[0, 1] and label in [0, 99].
|
||||
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5),
|
||||
'train',
|
||||
md5sum=CIFAR100_MD5,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Cifar100",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test100():
|
||||
"""
|
||||
CIFAR-100 test set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is image pixels in
|
||||
[0, 1] and label in [0, 99].
|
||||
|
||||
:return: Test reader creator.
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5),
|
||||
'test',
|
||||
md5sum=CIFAR100_MD5,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Cifar10",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train10(cycle=False):
|
||||
"""
|
||||
CIFAR-10 training set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is image pixels in
|
||||
[0, 1] and label in [0, 9].
|
||||
|
||||
:param cycle: whether to cycle through the dataset
|
||||
:type cycle: bool
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5),
|
||||
'data_batch',
|
||||
cycle=cycle,
|
||||
md5sum=CIFAR10_MD5,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Cifar10",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test10(cycle=False):
|
||||
"""
|
||||
CIFAR-10 test set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is image pixels in
|
||||
[0, 1] and label in [0, 9].
|
||||
|
||||
:param cycle: whether to cycle through the dataset
|
||||
:type cycle: bool
|
||||
:return: Test reader creator.
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5),
|
||||
'test_batch',
|
||||
cycle=cycle,
|
||||
md5sum=CIFAR10_MD5,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Cifar10",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5)
|
||||
paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5)
|
||||
@@ -0,0 +1,230 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
|
||||
import errno
|
||||
import glob
|
||||
import hashlib
|
||||
import importlib
|
||||
import os
|
||||
import pickle
|
||||
import re
|
||||
import shutil
|
||||
import sys
|
||||
import tempfile
|
||||
|
||||
import httpx
|
||||
|
||||
import paddle
|
||||
import paddle.dataset
|
||||
|
||||
__all__ = []
|
||||
|
||||
HOME = os.path.expanduser('~')
|
||||
|
||||
# If the default HOME dir does not support writing, we
|
||||
# will create a temporary folder to store the cache files.
|
||||
if not os.access(HOME, os.W_OK):
|
||||
r"""
|
||||
gettempdir() return the name of the directory used for temporary files.
|
||||
On Windows, the directories C:\TEMP, C:\TMP, \TEMP, and \TMP, in that order.
|
||||
On all other platforms, the directories /tmp, /var/tmp, and /usr/tmp, in that order.
|
||||
For more details, please refer to https://docs.python.org/3/library/tempfile.html
|
||||
"""
|
||||
HOME = tempfile.gettempdir()
|
||||
|
||||
DATA_HOME = os.path.join(HOME, '.cache', 'paddle', 'dataset')
|
||||
|
||||
|
||||
# When running unit tests, there could be multiple processes that
|
||||
# trying to create DATA_HOME directory simultaneously, so we cannot
|
||||
# use a if condition to check for the existence of the directory;
|
||||
# instead, we use the filesystem as the synchronization mechanism by
|
||||
# catching returned errors.
|
||||
def must_mkdirs(path):
|
||||
try:
|
||||
os.makedirs(DATA_HOME)
|
||||
except OSError as exc:
|
||||
if exc.errno != errno.EEXIST:
|
||||
raise
|
||||
|
||||
|
||||
must_mkdirs(DATA_HOME)
|
||||
|
||||
|
||||
def md5file(fname):
|
||||
hash_md5 = hashlib.md5()
|
||||
f = open(fname, "rb")
|
||||
for chunk in iter(lambda: f.read(4096), b""):
|
||||
hash_md5.update(chunk)
|
||||
f.close()
|
||||
return hash_md5.hexdigest()
|
||||
|
||||
|
||||
def download(url, module_name, md5sum, save_name=None):
|
||||
module_name = re.match("^[a-zA-Z0-9_/\\-]+$", module_name).group()
|
||||
if isinstance(save_name, str):
|
||||
save_name = re.match(
|
||||
"^(?:(?!\\.\\.)[a-zA-Z0-9_/\\.-])+$", save_name
|
||||
).group()
|
||||
dirname = os.path.join(DATA_HOME, module_name)
|
||||
if not os.path.exists(dirname):
|
||||
os.makedirs(dirname)
|
||||
|
||||
filename = os.path.join(
|
||||
dirname, url.split('/')[-1] if save_name is None else save_name
|
||||
)
|
||||
|
||||
if os.path.exists(filename) and md5file(filename) == md5sum:
|
||||
return filename
|
||||
|
||||
retry = 0
|
||||
retry_limit = 3
|
||||
while not (os.path.exists(filename) and md5file(filename) == md5sum):
|
||||
if os.path.exists(filename):
|
||||
sys.stderr.write(f"file {md5file(filename)} md5 {md5sum}\n")
|
||||
if retry < retry_limit:
|
||||
retry += 1
|
||||
else:
|
||||
raise RuntimeError(
|
||||
f"Cannot download {url} within retry limit {retry_limit}"
|
||||
)
|
||||
sys.stderr.write(
|
||||
f"Cache file {filename} not found, downloading {url} \n"
|
||||
)
|
||||
sys.stderr.write("Begin to download\n")
|
||||
try:
|
||||
# (risemeup1):use httpx to replace requests
|
||||
with httpx.stream(
|
||||
"GET", url, timeout=None, follow_redirects=True
|
||||
) as r:
|
||||
total_length = r.headers.get('content-length')
|
||||
if total_length is None:
|
||||
with open(filename, 'wb') as f:
|
||||
shutil.copyfileobj(r.raw, f)
|
||||
else:
|
||||
with open(filename, 'wb') as f:
|
||||
chunk_size = 4096
|
||||
total_length = int(total_length)
|
||||
total_iter = total_length / chunk_size + 1
|
||||
log_interval = (
|
||||
total_iter // 20 if total_iter > 20 else 1
|
||||
)
|
||||
log_index = 0
|
||||
bar = paddle.hapi.progressbar.ProgressBar(
|
||||
total_iter, name='item'
|
||||
)
|
||||
for data in r.iter_bytes(chunk_size=chunk_size):
|
||||
f.write(data)
|
||||
log_index += 1
|
||||
bar.update(log_index, {})
|
||||
if log_index % log_interval == 0:
|
||||
bar.update(log_index)
|
||||
|
||||
except Exception as e:
|
||||
# re-try
|
||||
continue
|
||||
sys.stderr.write("\nDownload finished\n")
|
||||
sys.stdout.flush()
|
||||
return filename
|
||||
|
||||
|
||||
def fetch_all():
|
||||
for module_name in [
|
||||
x for x in dir(paddle.dataset) if not x.startswith("__")
|
||||
]:
|
||||
if "fetch" in dir(
|
||||
importlib.import_module(f"paddle.dataset.{module_name}")
|
||||
):
|
||||
importlib.import_module(f'paddle.dataset.{module_name}').fetch()
|
||||
|
||||
|
||||
def split(reader, line_count, suffix="%05d.pickle", dumper=pickle.dump):
|
||||
"""
|
||||
you can call the function as:
|
||||
|
||||
split(paddle.dataset.cifar.train10(), line_count=1000,
|
||||
suffix="imikolov-train-%05d.pickle")
|
||||
|
||||
the output files as:
|
||||
|
||||
|-imikolov-train-00000.pickle
|
||||
|-imikolov-train-00001.pickle
|
||||
|- ...
|
||||
|-imikolov-train-00480.pickle
|
||||
|
||||
:param reader: is a reader creator
|
||||
:param line_count: line count for each file
|
||||
:param suffix: the suffix for the output files, should contain "%d"
|
||||
means the id for each file. Default is "%05d.pickle"
|
||||
:param dumper: is a callable function that dump object to file, this
|
||||
function will be called as dumper(obj, f) and obj is the object
|
||||
will be dumped, f is a file object. Default is cPickle.dump.
|
||||
"""
|
||||
if not callable(dumper):
|
||||
raise TypeError("dumper should be callable.")
|
||||
lines = []
|
||||
index_f = 0
|
||||
for i, d in enumerate(reader()):
|
||||
lines.append(d)
|
||||
if i >= line_count and i % line_count == 0:
|
||||
with open(suffix % index_f, "w") as f:
|
||||
dumper(lines, f)
|
||||
lines = []
|
||||
index_f += 1
|
||||
if lines:
|
||||
with open(suffix % index_f, "w") as f:
|
||||
dumper(lines, f)
|
||||
|
||||
|
||||
def cluster_files_reader(
|
||||
files_pattern, trainer_count, trainer_id, loader=pickle.load
|
||||
):
|
||||
"""
|
||||
Create a reader that yield element from the given files, select
|
||||
a file set according trainer count and trainer_id
|
||||
|
||||
:param files_pattern: the files which generating by split(...)
|
||||
:param trainer_count: total trainer count
|
||||
:param trainer_id: the trainer rank id
|
||||
:param loader: is a callable function that load object from file, this
|
||||
function will be called as loader(f) and f is a file object.
|
||||
Default is cPickle.load
|
||||
"""
|
||||
|
||||
def reader():
|
||||
if not callable(loader):
|
||||
raise TypeError("loader should be callable.")
|
||||
file_list = glob.glob(files_pattern)
|
||||
file_list.sort()
|
||||
my_file_list = []
|
||||
for idx, fn in enumerate(file_list):
|
||||
if idx % trainer_count == trainer_id:
|
||||
print(f"append file: {fn}")
|
||||
my_file_list.append(fn)
|
||||
for fn in my_file_list:
|
||||
with open(fn, "r") as f:
|
||||
lines = loader(f)
|
||||
yield from lines
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
def _check_exists_and_download(path, url, md5, module_name, download=True):
|
||||
if path and os.path.exists(path):
|
||||
return path
|
||||
|
||||
if download:
|
||||
return paddle.dataset.common.download(url, module_name, md5)
|
||||
else:
|
||||
raise ValueError(f'{path} not exists and auto download disabled')
|
||||
@@ -0,0 +1,281 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Conll05 dataset.
|
||||
Paddle semantic role labeling Book and demo use this dataset as an example.
|
||||
Because Conll05 is not free in public, the default downloaded URL is test set
|
||||
of Conll05 (which is public). Users can change URL and MD5 to their Conll
|
||||
dataset. And a pre-trained word vector model based on Wikipedia corpus is used
|
||||
to initialize SRL model.
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import tarfile
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
DATA_URL = 'http://paddlemodels.bj.bcebos.com/conll05st/conll05st-tests.tar.gz'
|
||||
DATA_MD5 = '387719152ae52d60422c016e92a742fc'
|
||||
WORDDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FwordDict.txt'
|
||||
WORDDICT_MD5 = 'ea7fb7d4c75cc6254716f0177a506baa'
|
||||
VERBDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FverbDict.txt'
|
||||
VERBDICT_MD5 = '0d2977293bbb6cbefab5b0f97db1e77c'
|
||||
TRGDICT_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2FtargetDict.txt'
|
||||
TRGDICT_MD5 = 'd8c7f03ceb5fc2e5a0fa7503a4353751'
|
||||
EMB_URL = 'http://paddlemodels.bj.bcebos.com/conll05st%2Femb'
|
||||
EMB_MD5 = 'bf436eb0faa1f6f9103017f8be57cdb7'
|
||||
|
||||
UNK_IDX = 0
|
||||
|
||||
|
||||
def load_label_dict(filename):
|
||||
d = {}
|
||||
tag_dict = set()
|
||||
with open(filename, 'r') as f:
|
||||
for i, line in enumerate(f):
|
||||
line = line.strip()
|
||||
if line.startswith(("B-", "I-")):
|
||||
tag_dict.add(line[2:])
|
||||
index = 0
|
||||
for tag in tag_dict:
|
||||
d["B-" + tag] = index
|
||||
index += 1
|
||||
d["I-" + tag] = index
|
||||
index += 1
|
||||
d["O"] = index
|
||||
return d
|
||||
|
||||
|
||||
def load_dict(filename):
|
||||
d = {}
|
||||
with open(filename, 'r') as f:
|
||||
for i, line in enumerate(f):
|
||||
d[line.strip()] = i
|
||||
return d
|
||||
|
||||
|
||||
def corpus_reader(data_path, words_name, props_name):
|
||||
"""
|
||||
Read one corpus. It returns an iterator. Each element of
|
||||
this iterator is a tuple including sentence and labels. The sentence is
|
||||
consist of a list of word IDs. The labels include a list of label IDs.
|
||||
:return: a iterator of data.
|
||||
:rtype: iterator
|
||||
"""
|
||||
|
||||
def reader():
|
||||
tf = tarfile.open(data_path)
|
||||
wf = tf.extractfile(words_name)
|
||||
pf = tf.extractfile(props_name)
|
||||
with (
|
||||
gzip.GzipFile(fileobj=wf) as words_file,
|
||||
gzip.GzipFile(fileobj=pf) as props_file,
|
||||
):
|
||||
sentences = []
|
||||
labels = []
|
||||
one_seg = []
|
||||
for word, label in zip(words_file, props_file):
|
||||
word = word.strip().decode()
|
||||
label = label.strip().decode().split()
|
||||
|
||||
if len(label) == 0: # end of sentence
|
||||
for i in range(len(one_seg[0])):
|
||||
a_kind_label = [x[i] for x in one_seg]
|
||||
labels.append(a_kind_label)
|
||||
|
||||
if len(labels) >= 1:
|
||||
verb_list = []
|
||||
for x in labels[0]:
|
||||
if x != '-':
|
||||
verb_list.append(x)
|
||||
|
||||
for i, lbl in enumerate(labels[1:]):
|
||||
cur_tag = 'O'
|
||||
is_in_bracket = False
|
||||
lbl_seq = []
|
||||
verb_word = ''
|
||||
for l in lbl:
|
||||
if l == '*' and not is_in_bracket:
|
||||
lbl_seq.append('O')
|
||||
elif l == '*' and is_in_bracket:
|
||||
lbl_seq.append('I-' + cur_tag)
|
||||
elif l == '*)':
|
||||
lbl_seq.append('I-' + cur_tag)
|
||||
is_in_bracket = False
|
||||
elif l.find('(') != -1 and l.find(')') != -1:
|
||||
cur_tag = l[1 : l.find('*')]
|
||||
lbl_seq.append('B-' + cur_tag)
|
||||
is_in_bracket = False
|
||||
elif l.find('(') != -1 and l.find(')') == -1:
|
||||
cur_tag = l[1 : l.find('*')]
|
||||
lbl_seq.append('B-' + cur_tag)
|
||||
is_in_bracket = True
|
||||
else:
|
||||
raise RuntimeError(f'Unexpected label: {l}')
|
||||
|
||||
yield sentences, verb_list[i], lbl_seq
|
||||
|
||||
sentences = []
|
||||
labels = []
|
||||
one_seg = []
|
||||
else:
|
||||
sentences.append(word)
|
||||
one_seg.append(label)
|
||||
|
||||
pf.close()
|
||||
wf.close()
|
||||
tf.close()
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
def reader_creator(
|
||||
corpus_reader, word_dict=None, predicate_dict=None, label_dict=None
|
||||
):
|
||||
def reader():
|
||||
for sentence, predicate, labels in corpus_reader():
|
||||
sen_len = len(sentence)
|
||||
|
||||
verb_index = labels.index('B-V')
|
||||
mark = [0] * len(labels)
|
||||
if verb_index > 0:
|
||||
mark[verb_index - 1] = 1
|
||||
ctx_n1 = sentence[verb_index - 1]
|
||||
else:
|
||||
ctx_n1 = 'bos'
|
||||
|
||||
if verb_index > 1:
|
||||
mark[verb_index - 2] = 1
|
||||
ctx_n2 = sentence[verb_index - 2]
|
||||
else:
|
||||
ctx_n2 = 'bos'
|
||||
|
||||
mark[verb_index] = 1
|
||||
ctx_0 = sentence[verb_index]
|
||||
|
||||
if verb_index < len(labels) - 1:
|
||||
mark[verb_index + 1] = 1
|
||||
ctx_p1 = sentence[verb_index + 1]
|
||||
else:
|
||||
ctx_p1 = 'eos'
|
||||
|
||||
if verb_index < len(labels) - 2:
|
||||
mark[verb_index + 2] = 1
|
||||
ctx_p2 = sentence[verb_index + 2]
|
||||
else:
|
||||
ctx_p2 = 'eos'
|
||||
|
||||
word_idx = [word_dict.get(w, UNK_IDX) for w in sentence]
|
||||
|
||||
ctx_n2_idx = [word_dict.get(ctx_n2, UNK_IDX)] * sen_len
|
||||
ctx_n1_idx = [word_dict.get(ctx_n1, UNK_IDX)] * sen_len
|
||||
ctx_0_idx = [word_dict.get(ctx_0, UNK_IDX)] * sen_len
|
||||
ctx_p1_idx = [word_dict.get(ctx_p1, UNK_IDX)] * sen_len
|
||||
ctx_p2_idx = [word_dict.get(ctx_p2, UNK_IDX)] * sen_len
|
||||
|
||||
pred_idx = [predicate_dict.get(predicate)] * sen_len
|
||||
label_idx = [label_dict.get(w) for w in labels]
|
||||
|
||||
yield (
|
||||
word_idx,
|
||||
ctx_n2_idx,
|
||||
ctx_n1_idx,
|
||||
ctx_0_idx,
|
||||
ctx_p1_idx,
|
||||
ctx_p2_idx,
|
||||
pred_idx,
|
||||
mark,
|
||||
label_idx,
|
||||
)
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Conll05st",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def get_dict():
|
||||
"""
|
||||
Get the word, verb and label dictionary of Wikipedia corpus.
|
||||
"""
|
||||
word_dict = load_dict(
|
||||
paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5)
|
||||
)
|
||||
verb_dict = load_dict(
|
||||
paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5)
|
||||
)
|
||||
label_dict = load_label_dict(
|
||||
paddle.dataset.common.download(TRGDICT_URL, 'conll05st', TRGDICT_MD5)
|
||||
)
|
||||
return word_dict, verb_dict, label_dict
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Conll05st",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def get_embedding():
|
||||
"""
|
||||
Get the trained word vector based on Wikipedia corpus.
|
||||
"""
|
||||
return paddle.dataset.common.download(EMB_URL, 'conll05st', EMB_MD5)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Conll05st",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test():
|
||||
"""
|
||||
Conll05 test set creator.
|
||||
|
||||
Because the training dataset is not free, the test dataset is used for
|
||||
training. It returns a reader creator, each sample in the reader is nine
|
||||
features, including sentence sequence, predicate, predicate context,
|
||||
predicate context flag and tagged sequence.
|
||||
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
word_dict, verb_dict, label_dict = get_dict()
|
||||
reader = corpus_reader(
|
||||
paddle.dataset.common.download(DATA_URL, 'conll05st', DATA_MD5),
|
||||
words_name='conll05st-release/test.wsj/words/test.wsj.words.gz',
|
||||
props_name='conll05st-release/test.wsj/props/test.wsj.props.gz',
|
||||
)
|
||||
return reader_creator(reader, word_dict, verb_dict, label_dict)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Conll05st",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(WORDDICT_URL, 'conll05st', WORDDICT_MD5)
|
||||
paddle.dataset.common.download(VERBDICT_URL, 'conll05st', VERBDICT_MD5)
|
||||
paddle.dataset.common.download(TRGDICT_URL, 'conll05st', TRGDICT_MD5)
|
||||
paddle.dataset.common.download(EMB_URL, 'conll05st', EMB_MD5)
|
||||
paddle.dataset.common.download(DATA_URL, 'conll05st', DATA_MD5)
|
||||
@@ -0,0 +1,243 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
This module will download dataset from
|
||||
http://www.robots.ox.ac.uk/~vgg/data/flowers/102/index.html
|
||||
and parse train/test dataset into paddle reader creators.
|
||||
|
||||
This set contains images of flowers belonging to 102 different categories.
|
||||
The images were acquired by searching the web and taking pictures. There are a
|
||||
minimum of 40 images for each category.
|
||||
|
||||
The database was used in:
|
||||
|
||||
Nilsback, M-E. and Zisserman, A. Automated flower classification over a large
|
||||
number of classes.Proceedings of the Indian Conference on Computer Vision,
|
||||
Graphics and Image Processing (2008)
|
||||
http://www.robots.ox.ac.uk/~vgg/publications/papers/nilsback08.{pdf,ps.gz}.
|
||||
|
||||
"""
|
||||
|
||||
import functools
|
||||
import tarfile
|
||||
from multiprocessing import cpu_count
|
||||
|
||||
from paddle.dataset.image import load_image_bytes, simple_transform
|
||||
from paddle.reader import map_readers, xmap_readers
|
||||
from paddle.utils import deprecated, try_import
|
||||
|
||||
from .common import download
|
||||
|
||||
__all__ = []
|
||||
|
||||
DATA_URL = 'http://paddlemodels.bj.bcebos.com/flowers/102flowers.tgz'
|
||||
LABEL_URL = 'http://paddlemodels.bj.bcebos.com/flowers/imagelabels.mat'
|
||||
SETID_URL = 'http://paddlemodels.bj.bcebos.com/flowers/setid.mat'
|
||||
DATA_MD5 = '52808999861908f626f3c1f4e79d11fa'
|
||||
LABEL_MD5 = 'e0620be6f572b9609742df49c70aed4d'
|
||||
SETID_MD5 = 'a5357ecc9cb78c4bef273ce3793fc85c'
|
||||
# In official 'readme', tstid is the flag of test data
|
||||
# and trnid is the flag of train data. But test data is more than train data.
|
||||
# So we exchange the train data and test data.
|
||||
TRAIN_FLAG = 'tstid'
|
||||
TEST_FLAG = 'trnid'
|
||||
VALID_FLAG = 'valid'
|
||||
|
||||
|
||||
def default_mapper(is_train, sample):
|
||||
'''
|
||||
map image bytes data to type needed by model input layer
|
||||
'''
|
||||
img, label = sample
|
||||
img = load_image_bytes(img)
|
||||
img = simple_transform(
|
||||
img, 256, 224, is_train, mean=[103.94, 116.78, 123.68]
|
||||
)
|
||||
return img.flatten().astype('float32'), label
|
||||
|
||||
|
||||
train_mapper = functools.partial(default_mapper, True)
|
||||
test_mapper = functools.partial(default_mapper, False)
|
||||
|
||||
|
||||
def reader_creator(
|
||||
data_file,
|
||||
label_file,
|
||||
setid_file,
|
||||
dataset_name,
|
||||
mapper,
|
||||
buffered_size=1024,
|
||||
use_xmap=True,
|
||||
cycle=False,
|
||||
):
|
||||
'''
|
||||
1. read images from tar file and
|
||||
merge images into batch files in 102flowers.tgz_batch/
|
||||
2. get a reader to read sample from batch file
|
||||
|
||||
:param data_file: downloaded data file
|
||||
:type data_file: string
|
||||
:param label_file: downloaded label file
|
||||
:type label_file: string
|
||||
:param setid_file: downloaded setid file containing information
|
||||
about how to split dataset
|
||||
:type setid_file: string
|
||||
:param dataset_name: data set name (tstid|trnid|valid)
|
||||
:type dataset_name: string
|
||||
:param mapper: a function to map image bytes data to type
|
||||
needed by model input layer
|
||||
:type mapper: callable
|
||||
:param buffered_size: the size of buffer used to process images
|
||||
:type buffered_size: int
|
||||
:param cycle: whether to cycle through the dataset
|
||||
:type cycle: bool
|
||||
:return: data reader
|
||||
:rtype: callable
|
||||
'''
|
||||
|
||||
def reader():
|
||||
scio = try_import('scipy.io')
|
||||
|
||||
labels = scio.loadmat(label_file)['labels'][0]
|
||||
indexes = scio.loadmat(setid_file)[dataset_name][0]
|
||||
|
||||
img2label = {}
|
||||
for i in indexes:
|
||||
img = f"jpg/image_{i:05}.jpg"
|
||||
img2label[img] = labels[i - 1]
|
||||
|
||||
tf = tarfile.open(data_file)
|
||||
mems = tf.getmembers()
|
||||
file_id = 0
|
||||
for mem in mems:
|
||||
if mem.name in img2label:
|
||||
image = tf.extractfile(mem).read()
|
||||
label = img2label[mem.name]
|
||||
yield image, int(label) - 1
|
||||
|
||||
if use_xmap:
|
||||
return xmap_readers(mapper, reader, min(4, cpu_count()), buffered_size)
|
||||
else:
|
||||
return map_readers(mapper, reader)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Flowers",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train(mapper=train_mapper, buffered_size=1024, use_xmap=True, cycle=False):
|
||||
'''
|
||||
Create flowers training set reader.
|
||||
It returns a reader, each sample in the reader is
|
||||
image pixels in [0, 1] and label in [1, 102]
|
||||
translated from original color image by steps:
|
||||
1. resize to 256*256
|
||||
2. random crop to 224*224
|
||||
3. flatten
|
||||
:param mapper: a function to map sample.
|
||||
:type mapper: callable
|
||||
:param buffered_size: the size of buffer used to process images
|
||||
:type buffered_size: int
|
||||
:param cycle: whether to cycle through the dataset
|
||||
:type cycle: bool
|
||||
:return: train data reader
|
||||
:rtype: callable
|
||||
'''
|
||||
return reader_creator(
|
||||
download(DATA_URL, 'flowers', DATA_MD5),
|
||||
download(LABEL_URL, 'flowers', LABEL_MD5),
|
||||
download(SETID_URL, 'flowers', SETID_MD5),
|
||||
TRAIN_FLAG,
|
||||
mapper,
|
||||
buffered_size,
|
||||
use_xmap,
|
||||
cycle=cycle,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Flowers",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test(mapper=test_mapper, buffered_size=1024, use_xmap=True, cycle=False):
|
||||
'''
|
||||
Create flowers test set reader.
|
||||
It returns a reader, each sample in the reader is
|
||||
image pixels in [0, 1] and label in [1, 102]
|
||||
translated from original color image by steps:
|
||||
1. resize to 256*256
|
||||
2. random crop to 224*224
|
||||
3. flatten
|
||||
:param mapper: a function to map sample.
|
||||
:type mapper: callable
|
||||
:param buffered_size: the size of buffer used to process images
|
||||
:type buffered_size: int
|
||||
:param cycle: whether to cycle through the dataset
|
||||
:type cycle: bool
|
||||
:return: test data reader
|
||||
:rtype: callable
|
||||
'''
|
||||
return reader_creator(
|
||||
download(DATA_URL, 'flowers', DATA_MD5),
|
||||
download(LABEL_URL, 'flowers', LABEL_MD5),
|
||||
download(SETID_URL, 'flowers', SETID_MD5),
|
||||
TEST_FLAG,
|
||||
mapper,
|
||||
buffered_size,
|
||||
use_xmap,
|
||||
cycle=cycle,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.Flowers",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def valid(mapper=test_mapper, buffered_size=1024, use_xmap=True):
|
||||
'''
|
||||
Create flowers validation set reader.
|
||||
It returns a reader, each sample in the reader is
|
||||
image pixels in [0, 1] and label in [1, 102]
|
||||
translated from original color image by steps:
|
||||
1. resize to 256*256
|
||||
2. random crop to 224*224
|
||||
3. flatten
|
||||
:param mapper: a function to map sample.
|
||||
:type mapper: callable
|
||||
:param buffered_size: the size of buffer used to process images
|
||||
:type buffered_size: int
|
||||
:return: test data reader
|
||||
:rtype: callable
|
||||
'''
|
||||
return reader_creator(
|
||||
download(DATA_URL, 'flowers', DATA_MD5),
|
||||
download(LABEL_URL, 'flowers', LABEL_MD5),
|
||||
download(SETID_URL, 'flowers', SETID_MD5),
|
||||
VALID_FLAG,
|
||||
mapper,
|
||||
buffered_size,
|
||||
use_xmap,
|
||||
)
|
||||
|
||||
|
||||
def fetch():
|
||||
download(DATA_URL, 'flowers', DATA_MD5)
|
||||
download(LABEL_URL, 'flowers', LABEL_MD5)
|
||||
download(SETID_URL, 'flowers', SETID_MD5)
|
||||
Executable
+391
@@ -0,0 +1,391 @@
|
||||
# Copyright (c) 2018 PaddlePaddle Authors. All Rights Reserved.
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
This file contains some common interfaces for image preprocess.
|
||||
Many users are confused about the image layout. We introduce
|
||||
the image layout as follows.
|
||||
|
||||
- CHW Layout
|
||||
|
||||
- The abbreviations: C=channel, H=Height, W=Width
|
||||
- The default layout of image opened by cv2 or PIL is HWC.
|
||||
PaddlePaddle only supports the CHW layout. And CHW is simply
|
||||
a transpose of HWC. It must transpose the input image.
|
||||
|
||||
- Color format: RGB or BGR
|
||||
|
||||
OpenCV use BGR color format. PIL use RGB color format. Both
|
||||
formats can be used for training. Noted that, the format should
|
||||
be keep consistent between the training and inference period.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
|
||||
import numpy as np
|
||||
|
||||
try:
|
||||
import cv2
|
||||
except ImportError:
|
||||
cv2 = None
|
||||
import pickle
|
||||
|
||||
__all__ = []
|
||||
|
||||
|
||||
def _check_cv2():
|
||||
if cv2 is None:
|
||||
import sys
|
||||
|
||||
sys.stderr.write(
|
||||
'''Warning with paddle image module: opencv-python should be imported,
|
||||
or paddle image module could NOT work; please install opencv-python first.'''
|
||||
)
|
||||
return False
|
||||
else:
|
||||
return True
|
||||
|
||||
|
||||
def batch_images_from_tar(
|
||||
data_file, dataset_name, img2label, num_per_batch=1024
|
||||
):
|
||||
"""
|
||||
Read images from tar file and batch them into batch file.
|
||||
|
||||
:param data_file: path of image tar file
|
||||
:type data_file: string
|
||||
:param dataset_name: 'train','test' or 'valid'
|
||||
:type dataset_name: string
|
||||
:param img2label: a dict with image file name as key
|
||||
and image's label as value
|
||||
:type img2label: dict
|
||||
:param num_per_batch: image number per batch file
|
||||
:type num_per_batch: int
|
||||
:return: path of list file containing paths of batch file
|
||||
:rtype: string
|
||||
"""
|
||||
batch_dir = data_file + "_batch"
|
||||
out_path = f"{batch_dir}/{dataset_name}_{os.getpid()}"
|
||||
meta_file = f"{batch_dir}/{dataset_name}_{os.getpid()}.txt"
|
||||
|
||||
if os.path.exists(out_path):
|
||||
return meta_file
|
||||
else:
|
||||
os.makedirs(out_path)
|
||||
|
||||
tf = tarfile.open(data_file)
|
||||
mems = tf.getmembers()
|
||||
data = []
|
||||
labels = []
|
||||
file_id = 0
|
||||
for mem in mems:
|
||||
if mem.name in img2label:
|
||||
data.append(tf.extractfile(mem).read())
|
||||
labels.append(img2label[mem.name])
|
||||
if len(data) == num_per_batch:
|
||||
output = {'label': labels, 'data': data}
|
||||
pickle.dump(
|
||||
output,
|
||||
open(f'{out_path}/batch_{file_id}', 'wb'),
|
||||
protocol=2,
|
||||
)
|
||||
file_id += 1
|
||||
data = []
|
||||
labels = []
|
||||
if len(data) > 0:
|
||||
output = {'label': labels, 'data': data}
|
||||
pickle.dump(
|
||||
output, open(f'{out_path}/batch_{file_id}', 'wb'), protocol=2
|
||||
)
|
||||
|
||||
with open(meta_file, mode='a') as meta:
|
||||
meta.writelines(
|
||||
os.path.abspath(f"{out_path}/{file}") + "\n"
|
||||
for file in os.listdir(out_path)
|
||||
)
|
||||
return meta_file
|
||||
|
||||
|
||||
def load_image_bytes(bytes, is_color=True):
|
||||
"""
|
||||
Load an color or gray image from bytes array.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> with open('cat.jpg') as f:
|
||||
... im = load_image_bytes(f.read())
|
||||
:param bytes: the input image bytes array.
|
||||
:type bytes: str
|
||||
:param is_color: If set is_color True, it will load and
|
||||
return a color image. Otherwise, it will
|
||||
load and return a gray image.
|
||||
:type is_color: bool
|
||||
"""
|
||||
assert _check_cv2() is True
|
||||
flag = 1 if is_color else 0
|
||||
file_bytes = np.asarray(bytearray(bytes), dtype=np.uint8)
|
||||
img = cv2.imdecode(file_bytes, flag)
|
||||
return img
|
||||
|
||||
|
||||
def load_image(file, is_color=True):
|
||||
"""
|
||||
Load an color or gray image from the file path.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_image('cat.jpg')
|
||||
|
||||
:param file: the input image path.
|
||||
:type file: string
|
||||
:param is_color: If set is_color True, it will load and
|
||||
return a color image. Otherwise, it will
|
||||
load and return a gray image.
|
||||
:type is_color: bool
|
||||
"""
|
||||
assert _check_cv2() is True
|
||||
|
||||
# cv2.IMAGE_COLOR for OpenCV3
|
||||
# cv2.CV_LOAD_IMAGE_COLOR for older OpenCV Version
|
||||
# cv2.IMAGE_GRAYSCALE for OpenCV3
|
||||
# cv2.CV_LOAD_IMAGE_GRAYSCALE for older OpenCV Version
|
||||
# Here, use constant 1 and 0
|
||||
# 1: COLOR, 0: GRAYSCALE
|
||||
flag = 1 if is_color else 0
|
||||
im = cv2.imread(file.encode('utf-8').decode('utf-8'), flag)
|
||||
return im
|
||||
|
||||
|
||||
def resize_short(im, size):
|
||||
"""
|
||||
Resize an image so that the length of shorter edge is size.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_image('cat.jpg')
|
||||
>>> im = resize_short(im, 256)
|
||||
|
||||
:param im: the input image with HWC layout.
|
||||
:type im: ndarray
|
||||
:param size: the shorter edge size of image after resizing.
|
||||
:type size: int
|
||||
"""
|
||||
assert _check_cv2() is True
|
||||
|
||||
h, w = im.shape[:2]
|
||||
h_new, w_new = size, size
|
||||
if h > w:
|
||||
h_new = size * h // w
|
||||
else:
|
||||
w_new = size * w // h
|
||||
im = cv2.resize(im, (w_new, h_new), interpolation=cv2.INTER_CUBIC)
|
||||
return im
|
||||
|
||||
|
||||
def to_chw(im, order=(2, 0, 1)):
|
||||
"""
|
||||
Transpose the input image order. The image layout is HWC format
|
||||
opened by cv2 or PIL. Transpose the input image to CHW layout
|
||||
according the order (2,0,1).
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_image('cat.jpg')
|
||||
>>> im = resize_short(im, 256)
|
||||
>>> im = to_chw(im)
|
||||
|
||||
:param im: the input image with HWC layout.
|
||||
:type im: ndarray
|
||||
:param order: the transposed order.
|
||||
:type order: tuple|list
|
||||
"""
|
||||
assert len(im.shape) == len(order)
|
||||
im = im.transpose(order)
|
||||
return im
|
||||
|
||||
|
||||
def center_crop(im, size, is_color=True):
|
||||
"""
|
||||
Crop the center of image with size.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_image('cat.jpg')
|
||||
>>> im = center_crop(im, 224)
|
||||
|
||||
:param im: the input image with HWC layout.
|
||||
:type im: ndarray
|
||||
:param size: the cropping size.
|
||||
:type size: int
|
||||
:param is_color: whether the image is color or not.
|
||||
:type is_color: bool
|
||||
"""
|
||||
h, w = im.shape[:2]
|
||||
h_start = (h - size) // 2
|
||||
w_start = (w - size) // 2
|
||||
h_end, w_end = h_start + size, w_start + size
|
||||
if is_color:
|
||||
im = im[h_start:h_end, w_start:w_end, :]
|
||||
else:
|
||||
im = im[h_start:h_end, w_start:w_end]
|
||||
return im
|
||||
|
||||
|
||||
def random_crop(im, size, is_color=True):
|
||||
"""
|
||||
Randomly crop input image with size.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_image('cat.jpg')
|
||||
>>> im = random_crop(im, 224)
|
||||
|
||||
:param im: the input image with HWC layout.
|
||||
:type im: ndarray
|
||||
:param size: the cropping size.
|
||||
:type size: int
|
||||
:param is_color: whether the image is color or not.
|
||||
:type is_color: bool
|
||||
"""
|
||||
h, w = im.shape[:2]
|
||||
h_start = np.random.randint(0, h - size + 1)
|
||||
w_start = np.random.randint(0, w - size + 1)
|
||||
h_end, w_end = h_start + size, w_start + size
|
||||
if is_color:
|
||||
im = im[h_start:h_end, w_start:w_end, :]
|
||||
else:
|
||||
im = im[h_start:h_end, w_start:w_end]
|
||||
return im
|
||||
|
||||
|
||||
def left_right_flip(im, is_color=True):
|
||||
"""
|
||||
Flip an image along the horizontal direction.
|
||||
Return the flipped image.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_image('cat.jpg')
|
||||
>>> im = left_right_flip(im)
|
||||
|
||||
:param im: input image with HWC layout or HW layout for gray image
|
||||
:type im: ndarray
|
||||
:param is_color: whether input image is color or not
|
||||
:type is_color: bool
|
||||
"""
|
||||
if len(im.shape) == 3 and is_color:
|
||||
return im[:, ::-1, :]
|
||||
else:
|
||||
return im[:, ::-1]
|
||||
|
||||
|
||||
def simple_transform(
|
||||
im, resize_size, crop_size, is_train, is_color=True, mean=None
|
||||
):
|
||||
"""
|
||||
Simply data argumentation for training. These operations include
|
||||
resizing, cropping and flipping.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_image('cat.jpg')
|
||||
>>> im = simple_transform(im, 256, 224, True)
|
||||
|
||||
:param im: The input image with HWC layout.
|
||||
:type im: ndarray
|
||||
:param resize_size: The shorter edge length of the resized image.
|
||||
:type resize_size: int
|
||||
:param crop_size: The cropping size.
|
||||
:type crop_size: int
|
||||
:param is_train: Whether it is training or not.
|
||||
:type is_train: bool
|
||||
:param is_color: whether the image is color or not.
|
||||
:type is_color: bool
|
||||
:param mean: the mean values, which can be element-wise mean values or
|
||||
mean values per channel.
|
||||
:type mean: numpy array | list
|
||||
"""
|
||||
im = resize_short(im, resize_size)
|
||||
if is_train:
|
||||
im = random_crop(im, crop_size, is_color=is_color)
|
||||
if np.random.randint(2) == 0:
|
||||
im = left_right_flip(im, is_color)
|
||||
else:
|
||||
im = center_crop(im, crop_size, is_color=is_color)
|
||||
if len(im.shape) == 3:
|
||||
im = to_chw(im)
|
||||
|
||||
im = im.astype('float32')
|
||||
if mean is not None:
|
||||
mean = np.array(mean, dtype=np.float32)
|
||||
# mean value, may be one value per channel
|
||||
if mean.ndim == 1 and is_color:
|
||||
mean = mean[:, np.newaxis, np.newaxis]
|
||||
elif mean.ndim == 1:
|
||||
mean = mean
|
||||
else:
|
||||
# elementwise mean
|
||||
assert len(mean.shape) == len(im)
|
||||
im -= mean
|
||||
|
||||
return im
|
||||
|
||||
|
||||
def load_and_transform(
|
||||
filename, resize_size, crop_size, is_train, is_color=True, mean=None
|
||||
):
|
||||
"""
|
||||
Load image from the input file `filename` and transform image for
|
||||
data argumentation. Please refer to the `simple_transform` interface
|
||||
for the transform operations.
|
||||
|
||||
Example usage:
|
||||
|
||||
.. code-block:: pycon
|
||||
|
||||
>>> im = load_and_transform('cat.jpg', 256, 224, True)
|
||||
|
||||
:param filename: The file name of input image.
|
||||
:type filename: string
|
||||
:param resize_size: The shorter edge length of the resized image.
|
||||
:type resize_size: int
|
||||
:param crop_size: The cropping size.
|
||||
:type crop_size: int
|
||||
:param is_train: Whether it is training or not.
|
||||
:type is_train: bool
|
||||
:param is_color: whether the image is color or not.
|
||||
:type is_color: bool
|
||||
:param mean: the mean values, which can be element-wise mean values or
|
||||
mean values per channel.
|
||||
:type mean: numpy array | list
|
||||
"""
|
||||
im = load_image(filename, is_color)
|
||||
im = simple_transform(im, resize_size, crop_size, is_train, is_color, mean)
|
||||
return im
|
||||
@@ -0,0 +1,181 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
IMDB dataset.
|
||||
|
||||
This module downloads IMDB dataset from
|
||||
http://ai.stanford.edu/%7Eamaas/data/sentiment/. This dataset contains a set
|
||||
of 25,000 highly polar movie reviews for training, and 25,000 for testing.
|
||||
Besides, this module also provides API for building dictionary.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import re
|
||||
import string
|
||||
import tarfile
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
# URL = 'http://ai.stanford.edu/%7Eamaas/data/sentiment/aclImdb_v1.tar.gz'
|
||||
URL = 'https://dataset.bj.bcebos.com/imdb%2FaclImdb_v1.tar.gz'
|
||||
MD5 = '7c2ac02c03563afcf9b574c7e56c153a'
|
||||
|
||||
|
||||
def tokenize(pattern):
|
||||
"""
|
||||
Read files that match the given pattern. Tokenize and yield each file.
|
||||
"""
|
||||
|
||||
with tarfile.open(paddle.dataset.common.download(URL, 'imdb', MD5)) as tarf:
|
||||
# Note that we should use tarfile.next(), which does
|
||||
# sequential access of member files, other than
|
||||
# tarfile.extractfile, which does random access and might
|
||||
# destroy hard disks.
|
||||
tf = tarf.next()
|
||||
while tf is not None:
|
||||
if bool(pattern.match(tf.name)):
|
||||
# newline and punctuations removal and ad-hoc tokenization.
|
||||
yield (
|
||||
tarf.extractfile(tf)
|
||||
.read()
|
||||
.rstrip(b'\n\r')
|
||||
.translate(None, string.punctuation.encode('latin-1'))
|
||||
.lower()
|
||||
.split()
|
||||
)
|
||||
tf = tarf.next()
|
||||
|
||||
|
||||
def build_dict(pattern, cutoff):
|
||||
"""
|
||||
Build a word dictionary from the corpus. Keys of the dictionary are words,
|
||||
and values are zero-based IDs of these words.
|
||||
"""
|
||||
word_freq = collections.defaultdict(int)
|
||||
for doc in tokenize(pattern):
|
||||
for word in doc:
|
||||
word_freq[word] += 1
|
||||
|
||||
# Not sure if we should prune less-frequent words here.
|
||||
word_freq = [x for x in word_freq.items() if x[1] > cutoff]
|
||||
|
||||
dictionary = sorted(word_freq, key=lambda x: (-x[1], x[0]))
|
||||
words, _ = list(zip(*dictionary))
|
||||
word_idx = dict(list(zip(words, range(len(words)))))
|
||||
word_idx['<unk>'] = len(words)
|
||||
return word_idx
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imdb",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def reader_creator(pos_pattern, neg_pattern, word_idx):
|
||||
UNK = word_idx['<unk>']
|
||||
INS = []
|
||||
|
||||
def load(pattern, out, label):
|
||||
for doc in tokenize(pattern):
|
||||
out.append(([word_idx.get(w, UNK) for w in doc], label))
|
||||
|
||||
load(pos_pattern, INS, 0)
|
||||
load(neg_pattern, INS, 1)
|
||||
|
||||
def reader():
|
||||
yield from INS
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imdb",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train(word_idx):
|
||||
"""
|
||||
IMDB training set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is an zero-based ID
|
||||
sequence and label in [0, 1].
|
||||
|
||||
:param word_idx: word dictionary
|
||||
:type word_idx: dict
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
re.compile(r"aclImdb/train/pos/.*\.txt$"),
|
||||
re.compile(r"aclImdb/train/neg/.*\.txt$"),
|
||||
word_idx,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imdb",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test(word_idx):
|
||||
"""
|
||||
IMDB test set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is an zero-based ID
|
||||
sequence and label in [0, 1].
|
||||
|
||||
:param word_idx: word dictionary
|
||||
:type word_idx: dict
|
||||
:return: Test reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
re.compile(r"aclImdb/test/pos/.*\.txt$"),
|
||||
re.compile(r"aclImdb/test/neg/.*\.txt$"),
|
||||
word_idx,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imdb",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def word_dict():
|
||||
"""
|
||||
Build a word dictionary from the corpus.
|
||||
|
||||
:return: Word dictionary
|
||||
:rtype: dict
|
||||
"""
|
||||
return build_dict(
|
||||
re.compile(r"aclImdb/((train)|(test))/((pos)|(neg))/.*\.txt$"), 150
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imdb",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(URL, 'imdb', MD5)
|
||||
@@ -0,0 +1,177 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
imikolov's simple dataset.
|
||||
|
||||
This module will download dataset from
|
||||
http://www.fit.vutbr.cz/~imikolov/rnnlm/ and parse training set and test set
|
||||
into paddle reader creators.
|
||||
"""
|
||||
|
||||
import collections
|
||||
import tarfile
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
# URL = 'http://www.fit.vutbr.cz/~imikolov/rnnlm/simple-examples.tgz'
|
||||
URL = 'https://dataset.bj.bcebos.com/imikolov%2Fsimple-examples.tgz'
|
||||
MD5 = '30177ea32e27c525793142b6bf2c8e2d'
|
||||
|
||||
|
||||
class DataType:
|
||||
NGRAM = 1
|
||||
SEQ = 2
|
||||
|
||||
|
||||
def word_count(f, word_freq=None):
|
||||
if word_freq is None:
|
||||
word_freq = collections.defaultdict(int)
|
||||
|
||||
for l in f:
|
||||
for w in l.strip().split():
|
||||
word_freq[w] += 1
|
||||
word_freq['<s>'] += 1
|
||||
word_freq['<e>'] += 1
|
||||
|
||||
return word_freq
|
||||
|
||||
|
||||
def build_dict(min_word_freq=50):
|
||||
"""
|
||||
Build a word dictionary from the corpus, Keys of the dictionary are words,
|
||||
and values are zero-based IDs of these words.
|
||||
"""
|
||||
train_filename = './simple-examples/data/ptb.train.txt'
|
||||
test_filename = './simple-examples/data/ptb.valid.txt'
|
||||
with tarfile.open(
|
||||
paddle.dataset.common.download(
|
||||
paddle.dataset.imikolov.URL, 'imikolov', paddle.dataset.imikolov.MD5
|
||||
)
|
||||
) as tf:
|
||||
trainf = tf.extractfile(train_filename)
|
||||
testf = tf.extractfile(test_filename)
|
||||
word_freq = word_count(testf, word_count(trainf))
|
||||
if '<unk>' in word_freq:
|
||||
# remove <unk> for now, since we will set it as last index
|
||||
del word_freq['<unk>']
|
||||
|
||||
word_freq = [x for x in word_freq.items() if x[1] > min_word_freq]
|
||||
|
||||
word_freq_sorted = sorted(word_freq, key=lambda x: (-x[1], x[0]))
|
||||
words, _ = list(zip(*word_freq_sorted))
|
||||
word_idx = dict(list(zip(words, range(len(words)))))
|
||||
word_idx['<unk>'] = len(words)
|
||||
|
||||
return word_idx
|
||||
|
||||
|
||||
def reader_creator(filename, word_idx, n, data_type):
|
||||
def reader():
|
||||
with tarfile.open(
|
||||
paddle.dataset.common.download(
|
||||
paddle.dataset.imikolov.URL,
|
||||
'imikolov',
|
||||
paddle.dataset.imikolov.MD5,
|
||||
)
|
||||
) as tf:
|
||||
f = tf.extractfile(filename)
|
||||
|
||||
UNK = word_idx['<unk>']
|
||||
for l in f:
|
||||
if DataType.NGRAM == data_type:
|
||||
assert n > -1, 'Invalid gram length'
|
||||
l = ['<s>', *l.strip().split(), '<e>']
|
||||
if len(l) >= n:
|
||||
l = [word_idx.get(w, UNK) for w in l]
|
||||
for i in range(n, len(l) + 1):
|
||||
yield tuple(l[i - n : i])
|
||||
elif DataType.SEQ == data_type:
|
||||
l = l.strip().split()
|
||||
l = [word_idx.get(w, UNK) for w in l]
|
||||
src_seq = [word_idx['<s>'], *l]
|
||||
trg_seq = [*l, word_idx['<e>']]
|
||||
if n > 0 and len(src_seq) > n:
|
||||
continue
|
||||
yield src_seq, trg_seq
|
||||
else:
|
||||
raise AssertionError('Unknown data type')
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imikolov",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train(word_idx, n, data_type=DataType.NGRAM):
|
||||
"""
|
||||
imikolov training set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is a word ID
|
||||
tuple.
|
||||
|
||||
:param word_idx: word dictionary
|
||||
:type word_idx: dict
|
||||
:param n: sliding window size if type is ngram, otherwise max length of sequence
|
||||
:type n: int
|
||||
:param data_type: data type (ngram or sequence)
|
||||
:type data_type: member variable of DataType (NGRAM or SEQ)
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
'./simple-examples/data/ptb.train.txt', word_idx, n, data_type
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imikolov",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test(word_idx, n, data_type=DataType.NGRAM):
|
||||
"""
|
||||
imikolov test set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is a word ID
|
||||
tuple.
|
||||
|
||||
:param word_idx: word dictionary
|
||||
:type word_idx: dict
|
||||
:param n: sliding window size if type is ngram, otherwise max length of sequence
|
||||
:type n: int
|
||||
:param data_type: data type (ngram or sequence)
|
||||
:type data_type: member variable of DataType (NGRAM or SEQ)
|
||||
:return: Test reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
'./simple-examples/data/ptb.valid.txt', word_idx, n, data_type
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Imikolov",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(URL, "imikolov", MD5)
|
||||
@@ -0,0 +1,156 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
MNIST dataset.
|
||||
|
||||
This module will download dataset from http://yann.lecun.com/exdb/mnist/ and
|
||||
parse training set and test set into paddle reader creators.
|
||||
"""
|
||||
|
||||
import gzip
|
||||
import struct
|
||||
|
||||
import numpy
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
URL_PREFIX = 'https://dataset.bj.bcebos.com/mnist/'
|
||||
TEST_IMAGE_URL = URL_PREFIX + 't10k-images-idx3-ubyte.gz'
|
||||
TEST_IMAGE_MD5 = '9fb629c4189551a2d022fa330f9573f3'
|
||||
TEST_LABEL_URL = URL_PREFIX + 't10k-labels-idx1-ubyte.gz'
|
||||
TEST_LABEL_MD5 = 'ec29112dd5afa0611ce80d1b7f02629c'
|
||||
TRAIN_IMAGE_URL = URL_PREFIX + 'train-images-idx3-ubyte.gz'
|
||||
TRAIN_IMAGE_MD5 = 'f68b3c2dcbeaaa9fbdd348bbdeb94873'
|
||||
TRAIN_LABEL_URL = URL_PREFIX + 'train-labels-idx1-ubyte.gz'
|
||||
TRAIN_LABEL_MD5 = 'd53e105ee54ea40749a09fcbcd1e9432'
|
||||
|
||||
|
||||
def reader_creator(image_filename, label_filename, buffer_size):
|
||||
def reader():
|
||||
with gzip.GzipFile(image_filename, 'rb') as image_file:
|
||||
img_buf = image_file.read()
|
||||
with gzip.GzipFile(label_filename, 'rb') as label_file:
|
||||
lab_buf = label_file.read()
|
||||
|
||||
step_label = 0
|
||||
|
||||
offset_img = 0
|
||||
# read from Big-endian
|
||||
# get file info from magic byte
|
||||
# image file : 16B
|
||||
magic_byte_img = '>IIII'
|
||||
magic_img, image_num, rows, cols = struct.unpack_from(
|
||||
magic_byte_img, img_buf, offset_img
|
||||
)
|
||||
offset_img += struct.calcsize(magic_byte_img)
|
||||
|
||||
offset_lab = 0
|
||||
# label file : 8B
|
||||
magic_byte_lab = '>II'
|
||||
magic_lab, label_num = struct.unpack_from(
|
||||
magic_byte_lab, lab_buf, offset_lab
|
||||
)
|
||||
offset_lab += struct.calcsize(magic_byte_lab)
|
||||
|
||||
while True:
|
||||
if step_label >= label_num:
|
||||
break
|
||||
fmt_label = '>' + str(buffer_size) + 'B'
|
||||
labels = struct.unpack_from(fmt_label, lab_buf, offset_lab)
|
||||
offset_lab += struct.calcsize(fmt_label)
|
||||
step_label += buffer_size
|
||||
|
||||
fmt_images = '>' + str(buffer_size * rows * cols) + 'B'
|
||||
images_temp = struct.unpack_from(
|
||||
fmt_images, img_buf, offset_img
|
||||
)
|
||||
images = numpy.reshape(
|
||||
images_temp, (buffer_size, rows * cols)
|
||||
).astype('float32')
|
||||
offset_img += struct.calcsize(fmt_images)
|
||||
|
||||
images = images / 255.0
|
||||
images = images * 2.0
|
||||
images = images - 1.0
|
||||
|
||||
for i in range(buffer_size):
|
||||
yield images[i, :], int(labels[i])
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.MNIST",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train():
|
||||
"""
|
||||
MNIST training set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is image pixels in
|
||||
[-1, 1] and label in [0, 9].
|
||||
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(
|
||||
TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5
|
||||
),
|
||||
paddle.dataset.common.download(
|
||||
TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5
|
||||
),
|
||||
100,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.MNIST",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test():
|
||||
"""
|
||||
MNIST test set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is image pixels in
|
||||
[-1, 1] and label in [0, 9].
|
||||
|
||||
:return: Test reader creator.
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(TEST_IMAGE_URL, 'mnist', TEST_IMAGE_MD5),
|
||||
paddle.dataset.common.download(TEST_LABEL_URL, 'mnist', TEST_LABEL_MD5),
|
||||
100,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.MNIST",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(TRAIN_IMAGE_URL, 'mnist', TRAIN_IMAGE_MD5)
|
||||
paddle.dataset.common.download(TRAIN_LABEL_URL, 'mnist', TRAIN_LABEL_MD5)
|
||||
paddle.dataset.common.download(TEST_IMAGE_URL, 'mnist', TEST_IMAGE_MD5)
|
||||
paddle.dataset.common.download(TEST_LABEL_URL, 'mnist', TEST_LABEL_MD5)
|
||||
@@ -0,0 +1,316 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Movielens 1-M dataset.
|
||||
|
||||
Movielens 1-M dataset contains 1 million ratings from 6000 users on 4000
|
||||
movies, which was collected by GroupLens Research. This module will download
|
||||
Movielens 1-M dataset from
|
||||
http://files.grouplens.org/datasets/movielens/ml-1m.zip and parse training
|
||||
set and test set into paddle reader creators.
|
||||
|
||||
"""
|
||||
|
||||
import functools
|
||||
import re
|
||||
import zipfile
|
||||
|
||||
import numpy as np
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
age_table = [1, 18, 25, 35, 45, 50, 56]
|
||||
|
||||
# URL = 'http://files.grouplens.org/datasets/movielens/ml-1m.zip'
|
||||
URL = 'https://dataset.bj.bcebos.com/movielens%2Fml-1m.zip'
|
||||
MD5 = 'c4d9eecfca2ab87c1945afe126590906'
|
||||
|
||||
|
||||
class MovieInfo:
|
||||
"""
|
||||
Movie id, title and categories information are stored in MovieInfo.
|
||||
"""
|
||||
|
||||
def __init__(self, index, categories, title):
|
||||
self.index = int(index)
|
||||
self.categories = categories
|
||||
self.title = title
|
||||
|
||||
def value(self):
|
||||
"""
|
||||
Get information from a movie.
|
||||
"""
|
||||
return [
|
||||
self.index,
|
||||
[CATEGORIES_DICT[c] for c in self.categories],
|
||||
[MOVIE_TITLE_DICT[w.lower()] for w in self.title.split()],
|
||||
]
|
||||
|
||||
def __str__(self):
|
||||
return f"<MovieInfo id({self.index}), title({self.title}), categories({self.categories})>"
|
||||
|
||||
def __repr__(self):
|
||||
return self.__str__()
|
||||
|
||||
|
||||
class UserInfo:
|
||||
"""
|
||||
User id, gender, age, and job information are stored in UserInfo.
|
||||
"""
|
||||
|
||||
def __init__(self, index, gender, age, job_id):
|
||||
self.index = int(index)
|
||||
self.is_male = gender == 'M'
|
||||
self.age = age_table.index(int(age))
|
||||
self.job_id = int(job_id)
|
||||
|
||||
def value(self):
|
||||
"""
|
||||
Get information from a user.
|
||||
"""
|
||||
return [self.index, 0 if self.is_male else 1, self.age, self.job_id]
|
||||
|
||||
def __str__(self):
|
||||
gender = "M" if self.is_male else "F"
|
||||
return f"<UserInfo id({self.index}), gender({gender}), age({age_table[self.age]}), job({self.job_id})>"
|
||||
|
||||
def __repr__(self):
|
||||
return str(self)
|
||||
|
||||
|
||||
MOVIE_INFO = None
|
||||
MOVIE_TITLE_DICT = None
|
||||
CATEGORIES_DICT = None
|
||||
USER_INFO = None
|
||||
|
||||
|
||||
def __initialize_meta_info__():
|
||||
fn = paddle.dataset.common.download(URL, "movielens", MD5)
|
||||
global MOVIE_INFO
|
||||
if MOVIE_INFO is None:
|
||||
pattern = re.compile(r'^(.*)\((\d+)\)$')
|
||||
with zipfile.ZipFile(file=fn) as package:
|
||||
for info in package.infolist():
|
||||
assert isinstance(info, zipfile.ZipInfo)
|
||||
MOVIE_INFO = {}
|
||||
title_word_set = set()
|
||||
categories_set = set()
|
||||
with package.open('ml-1m/movies.dat') as movie_file:
|
||||
for i, line in enumerate(movie_file):
|
||||
line = line.decode(encoding='latin')
|
||||
movie_id, title, categories = line.strip().split('::')
|
||||
categories = categories.split('|')
|
||||
for c in categories:
|
||||
categories_set.add(c)
|
||||
title = pattern.match(title).group(1)
|
||||
MOVIE_INFO[int(movie_id)] = MovieInfo(
|
||||
index=movie_id, categories=categories, title=title
|
||||
)
|
||||
for w in title.split():
|
||||
title_word_set.add(w.lower())
|
||||
|
||||
global MOVIE_TITLE_DICT
|
||||
MOVIE_TITLE_DICT = {}
|
||||
for i, w in enumerate(title_word_set):
|
||||
MOVIE_TITLE_DICT[w] = i
|
||||
|
||||
global CATEGORIES_DICT
|
||||
CATEGORIES_DICT = {}
|
||||
for i, c in enumerate(categories_set):
|
||||
CATEGORIES_DICT[c] = i
|
||||
|
||||
global USER_INFO
|
||||
USER_INFO = {}
|
||||
with package.open('ml-1m/users.dat') as user_file:
|
||||
for line in user_file:
|
||||
line = line.decode(encoding='latin')
|
||||
uid, gender, age, job, _ = line.strip().split("::")
|
||||
USER_INFO[int(uid)] = UserInfo(
|
||||
index=uid, gender=gender, age=age, job_id=job
|
||||
)
|
||||
return fn
|
||||
|
||||
|
||||
def __reader__(rand_seed=0, test_ratio=0.1, is_test=False):
|
||||
fn = __initialize_meta_info__()
|
||||
np.random.seed(rand_seed)
|
||||
with (
|
||||
zipfile.ZipFile(file=fn) as package,
|
||||
package.open('ml-1m/ratings.dat') as rating,
|
||||
):
|
||||
for line in rating:
|
||||
line = line.decode(encoding='latin')
|
||||
if (np.random.random() < test_ratio) == is_test:
|
||||
uid, mov_id, rating, _ = line.strip().split("::")
|
||||
uid = int(uid)
|
||||
mov_id = int(mov_id)
|
||||
rating = float(rating) * 2 - 5.0
|
||||
|
||||
mov = MOVIE_INFO[mov_id]
|
||||
usr = USER_INFO[uid]
|
||||
yield usr.value() + mov.value() + [[rating]]
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def __reader_creator__(**kwargs):
|
||||
return lambda: __reader__(**kwargs)
|
||||
|
||||
|
||||
train = functools.partial(__reader_creator__, is_test=False)
|
||||
test = functools.partial(__reader_creator__, is_test=True)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def get_movie_title_dict():
|
||||
"""
|
||||
Get movie title dictionary.
|
||||
"""
|
||||
__initialize_meta_info__()
|
||||
return MOVIE_TITLE_DICT
|
||||
|
||||
|
||||
def __max_index_info__(a, b):
|
||||
if a.index > b.index:
|
||||
return a
|
||||
else:
|
||||
return b
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def max_movie_id():
|
||||
"""
|
||||
Get the maximum value of movie id.
|
||||
"""
|
||||
__initialize_meta_info__()
|
||||
return functools.reduce(__max_index_info__, list(MOVIE_INFO.values())).index
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def max_user_id():
|
||||
"""
|
||||
Get the maximum value of user id.
|
||||
"""
|
||||
__initialize_meta_info__()
|
||||
return functools.reduce(__max_index_info__, list(USER_INFO.values())).index
|
||||
|
||||
|
||||
def __max_job_id_impl__(a, b):
|
||||
if a.job_id > b.job_id:
|
||||
return a
|
||||
else:
|
||||
return b
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def max_job_id():
|
||||
"""
|
||||
Get the maximum value of job id.
|
||||
"""
|
||||
__initialize_meta_info__()
|
||||
return functools.reduce(
|
||||
__max_job_id_impl__, list(USER_INFO.values())
|
||||
).job_id
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def movie_categories():
|
||||
"""
|
||||
Get movie categories dictionary.
|
||||
"""
|
||||
__initialize_meta_info__()
|
||||
return CATEGORIES_DICT
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def user_info():
|
||||
"""
|
||||
Get user info dictionary.
|
||||
"""
|
||||
__initialize_meta_info__()
|
||||
return USER_INFO
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def movie_info():
|
||||
"""
|
||||
Get movie info dictionary.
|
||||
"""
|
||||
__initialize_meta_info__()
|
||||
return MOVIE_INFO
|
||||
|
||||
|
||||
def unittest():
|
||||
for train_count, _ in enumerate(train()()):
|
||||
pass
|
||||
for test_count, _ in enumerate(test()()):
|
||||
pass
|
||||
|
||||
print(train_count, test_count)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.Movielens",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(URL, "movielens", MD5)
|
||||
|
||||
|
||||
if __name__ == '__main__':
|
||||
unittest()
|
||||
@@ -0,0 +1,190 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
UCI Housing dataset.
|
||||
|
||||
This module will download dataset from
|
||||
https://archive.ics.uci.edu/ml/machine-learning-databases/housing/ and
|
||||
parse training set and test set into paddle reader creators.
|
||||
"""
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
import tempfile
|
||||
|
||||
import numpy as np
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
URL = 'http://paddlemodels.bj.bcebos.com/uci_housing/housing.data'
|
||||
MD5 = 'd4accdce7a25600298819f8e28e8d593'
|
||||
feature_names = [
|
||||
'CRIM',
|
||||
'ZN',
|
||||
'INDUS',
|
||||
'CHAS',
|
||||
'NOX',
|
||||
'RM',
|
||||
'AGE',
|
||||
'DIS',
|
||||
'RAD',
|
||||
'TAX',
|
||||
'PTRATIO',
|
||||
'B',
|
||||
'LSTAT',
|
||||
]
|
||||
|
||||
UCI_TRAIN_DATA = None
|
||||
UCI_TEST_DATA = None
|
||||
|
||||
FLUID_URL_MODEL = 'https://github.com/PaddlePaddle/book/raw/develop/01.fit_a_line/fluid/fit_a_line.fluid.tar'
|
||||
FLUID_MD5_MODEL = '6e6dd637ccd5993961f68bfbde46090b'
|
||||
|
||||
|
||||
def feature_range(maximums, minimums):
|
||||
import matplotlib
|
||||
|
||||
matplotlib.use('Agg')
|
||||
import matplotlib.pyplot as plt
|
||||
|
||||
fig, ax = plt.subplots()
|
||||
feature_num = len(maximums)
|
||||
ax.bar(
|
||||
list(range(feature_num)), maximums - minimums, color='r', align='center'
|
||||
)
|
||||
ax.set_title('feature scale')
|
||||
plt.xticks(list(range(feature_num)), feature_names)
|
||||
plt.xlim([-1, feature_num])
|
||||
fig.set_figheight(6)
|
||||
fig.set_figwidth(10)
|
||||
if not os.path.exists('./image'):
|
||||
os.makedirs('./image')
|
||||
fig.savefig('image/ranges.png', dpi=48)
|
||||
plt.close(fig)
|
||||
|
||||
|
||||
def load_data(filename, feature_num=14, ratio=0.8):
|
||||
global UCI_TRAIN_DATA, UCI_TEST_DATA
|
||||
if UCI_TRAIN_DATA is not None and UCI_TEST_DATA is not None:
|
||||
return
|
||||
|
||||
data = np.fromfile(filename, sep=' ')
|
||||
data = data.reshape(data.shape[0] // feature_num, feature_num)
|
||||
maximums, minimums, avgs = (
|
||||
data.max(axis=0),
|
||||
data.min(axis=0),
|
||||
data.sum(axis=0) / data.shape[0],
|
||||
)
|
||||
# if you want to print the distribution of input data, you could use function of feature_range
|
||||
# feature_range(maximums[:-1], minimums[:-1])
|
||||
for i in range(feature_num - 1):
|
||||
data[:, i] = (data[:, i] - avgs[i]) / (maximums[i] - minimums[i])
|
||||
offset = int(data.shape[0] * ratio)
|
||||
UCI_TRAIN_DATA = data[:offset]
|
||||
UCI_TEST_DATA = data[offset:]
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.UCIHousing",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train():
|
||||
"""
|
||||
UCI_HOUSING training set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is features after
|
||||
normalization and price number.
|
||||
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
global UCI_TRAIN_DATA
|
||||
load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5))
|
||||
|
||||
def reader():
|
||||
for d in UCI_TRAIN_DATA:
|
||||
yield d[:-1], d[-1:]
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.UCIHousing",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test():
|
||||
"""
|
||||
UCI_HOUSING test set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is features after
|
||||
normalization and price number.
|
||||
|
||||
:return: Test reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
global UCI_TEST_DATA
|
||||
load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5))
|
||||
|
||||
def reader():
|
||||
for d in UCI_TEST_DATA:
|
||||
yield d[:-1], d[-1:]
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
def fluid_model():
|
||||
parameter_tar = paddle.dataset.common.download(
|
||||
FLUID_URL_MODEL, 'uci_housing', FLUID_MD5_MODEL, 'fit_a_line.fluid.tar'
|
||||
)
|
||||
|
||||
tar = tarfile.TarFile(parameter_tar, mode='r')
|
||||
dirpath = tempfile.mkdtemp()
|
||||
tar.extractall(path=dirpath)
|
||||
|
||||
return dirpath
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.UCIHousing",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def predict_reader():
|
||||
"""
|
||||
It returns just one tuple data to do inference.
|
||||
|
||||
:return: one tuple data
|
||||
:rtype: tuple
|
||||
"""
|
||||
global UCI_TEST_DATA
|
||||
load_data(paddle.dataset.common.download(URL, 'uci_housing', MD5))
|
||||
return (UCI_TEST_DATA[0][:-1],)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.UCIHousing",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(URL, 'uci_housing', MD5)
|
||||
@@ -0,0 +1,104 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
Image dataset for segmentation.
|
||||
The 2012 dataset contains images from 2008-2011 for which additional
|
||||
segmentations have been prepared. As in previous years the assignment
|
||||
to training/test sets has been maintained. The total number of images
|
||||
with segmentation has been increased from 7,062 to 9,993.
|
||||
"""
|
||||
|
||||
import io
|
||||
import tarfile
|
||||
|
||||
import numpy as np
|
||||
from PIL import Image
|
||||
|
||||
from paddle.dataset.common import download
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
VOC_URL = 'http://host.robots.ox.ac.uk/pascal/VOC/voc2012/\
|
||||
VOCtrainval_11-May-2012.tar'
|
||||
|
||||
VOC_MD5 = '6cd6e144f989b92b3379bac3b3de84fd'
|
||||
SET_FILE = 'VOCdevkit/VOC2012/ImageSets/Segmentation/{}.txt'
|
||||
DATA_FILE = 'VOCdevkit/VOC2012/JPEGImages/{}.jpg'
|
||||
LABEL_FILE = 'VOCdevkit/VOC2012/SegmentationClass/{}.png'
|
||||
|
||||
CACHE_DIR = 'voc2012'
|
||||
|
||||
|
||||
def reader_creator(filename, sub_name):
|
||||
tarobject = tarfile.open(filename)
|
||||
name2mem = {}
|
||||
for ele in tarobject.getmembers():
|
||||
name2mem[ele.name] = ele
|
||||
|
||||
def reader():
|
||||
set_file = SET_FILE.format(sub_name)
|
||||
sets = tarobject.extractfile(name2mem[set_file])
|
||||
for line in sets:
|
||||
line = line.strip()
|
||||
data_file = DATA_FILE.format(line)
|
||||
label_file = LABEL_FILE.format(line)
|
||||
data = tarobject.extractfile(name2mem[data_file]).read()
|
||||
label = tarobject.extractfile(name2mem[label_file]).read()
|
||||
data = Image.open(io.BytesIO(data))
|
||||
label = Image.open(io.BytesIO(label))
|
||||
data = np.array(data)
|
||||
label = np.array(label)
|
||||
yield data, label
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.VOC2012",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train():
|
||||
"""
|
||||
Create a train dataset reader containing 2913 images in HWC order.
|
||||
"""
|
||||
return reader_creator(download(VOC_URL, CACHE_DIR, VOC_MD5), 'trainval')
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.VOC2012",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test():
|
||||
"""
|
||||
Create a test dataset reader containing 1464 images in HWC order.
|
||||
"""
|
||||
return reader_creator(download(VOC_URL, CACHE_DIR, VOC_MD5), 'train')
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.vision.datasets.VOC2012",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def val():
|
||||
"""
|
||||
Create a val dataset reader containing 1449 images in HWC order.
|
||||
"""
|
||||
return reader_creator(download(VOC_URL, CACHE_DIR, VOC_MD5), 'val')
|
||||
@@ -0,0 +1,200 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
WMT14 dataset.
|
||||
The original WMT14 dataset is too large and a small set of data for set is
|
||||
provided. This module will download dataset from
|
||||
http://paddlepaddle.bj.bcebos.com/demo/wmt_shrinked_data/wmt14.tgz and
|
||||
parse training set and test set into paddle reader creators.
|
||||
|
||||
"""
|
||||
|
||||
import tarfile
|
||||
|
||||
import paddle.dataset.common
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
URL_DEV_TEST = (
|
||||
'http://www-lium.univ-lemans.fr/~schwenk/cslm_joint_paper/data/dev+test.tgz'
|
||||
)
|
||||
MD5_DEV_TEST = '7d7897317ddd8ba0ae5c5fa7248d3ff5'
|
||||
# this is a small set of data for test. The original data is too large and
|
||||
# will be add later.
|
||||
URL_TRAIN = 'http://paddlemodels.bj.bcebos.com/wmt/wmt14.tgz'
|
||||
MD5_TRAIN = '0791583d57d5beb693b9414c5b36798c'
|
||||
# BLEU of this trained model is 26.92
|
||||
URL_MODEL = 'http://paddlemodels.bj.bcebos.com/wmt%2Fwmt14.tgz'
|
||||
MD5_MODEL = '0cb4a5366189b6acba876491c8724fa3'
|
||||
|
||||
START = "<s>"
|
||||
END = "<e>"
|
||||
UNK = "<unk>"
|
||||
UNK_IDX = 2
|
||||
|
||||
|
||||
def __read_to_dict(tar_file, dict_size):
|
||||
def __to_dict(fd, size):
|
||||
out_dict = {}
|
||||
for line_count, line in enumerate(fd):
|
||||
if line_count < size:
|
||||
out_dict[line.strip().decode()] = line_count
|
||||
else:
|
||||
break
|
||||
return out_dict
|
||||
|
||||
with tarfile.open(tar_file, mode='r') as f:
|
||||
names = [
|
||||
each_item.name
|
||||
for each_item in f
|
||||
if each_item.name.endswith("src.dict")
|
||||
]
|
||||
assert len(names) == 1
|
||||
src_dict = __to_dict(f.extractfile(names[0]), dict_size)
|
||||
names = [
|
||||
each_item.name
|
||||
for each_item in f
|
||||
if each_item.name.endswith("trg.dict")
|
||||
]
|
||||
assert len(names) == 1
|
||||
trg_dict = __to_dict(f.extractfile(names[0]), dict_size)
|
||||
return src_dict, trg_dict
|
||||
|
||||
|
||||
def reader_creator(tar_file, file_name, dict_size):
|
||||
def reader():
|
||||
src_dict, trg_dict = __read_to_dict(tar_file, dict_size)
|
||||
with tarfile.open(tar_file, mode='r') as f:
|
||||
names = [
|
||||
each_item.name
|
||||
for each_item in f
|
||||
if each_item.name.endswith(file_name)
|
||||
]
|
||||
for name in names:
|
||||
for line in f.extractfile(name):
|
||||
line = line.decode()
|
||||
line_split = line.strip().split('\t')
|
||||
if len(line_split) != 2:
|
||||
continue
|
||||
src_seq = line_split[0] # one source sequence
|
||||
src_words = src_seq.split()
|
||||
src_ids = [
|
||||
src_dict.get(w, UNK_IDX)
|
||||
for w in [START, *src_words, END]
|
||||
]
|
||||
|
||||
trg_seq = line_split[1] # one target sequence
|
||||
trg_words = trg_seq.split()
|
||||
trg_ids = [trg_dict.get(w, UNK_IDX) for w in trg_words]
|
||||
|
||||
# remove sequence whose length > 80 in training mode
|
||||
if len(src_ids) > 80 or len(trg_ids) > 80:
|
||||
continue
|
||||
trg_ids_next = [*trg_ids, trg_dict[END]]
|
||||
trg_ids = [trg_dict[START], *trg_ids]
|
||||
|
||||
yield src_ids, trg_ids, trg_ids_next
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT14",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train(dict_size):
|
||||
"""
|
||||
WMT14 training set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is source language
|
||||
word ID sequence, target language word ID sequence and next word ID
|
||||
sequence.
|
||||
|
||||
:return: Training reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN),
|
||||
'train/train',
|
||||
dict_size,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT14",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test(dict_size):
|
||||
"""
|
||||
WMT14 test set creator.
|
||||
|
||||
It returns a reader creator, each sample in the reader is source language
|
||||
word ID sequence, target language word ID sequence and next word ID
|
||||
sequence.
|
||||
|
||||
:return: Test reader creator
|
||||
:rtype: callable
|
||||
"""
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN),
|
||||
'test/test',
|
||||
dict_size,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT14",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def gen(dict_size):
|
||||
return reader_creator(
|
||||
paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN),
|
||||
'gen/gen',
|
||||
dict_size,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT14",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def get_dict(dict_size, reverse=True):
|
||||
# if reverse = False, return dict = {'a':'001', 'b':'002', ...}
|
||||
# else reverse = true, return dict = {'001':'a', '002':'b', ...}
|
||||
tar_file = paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN)
|
||||
src_dict, trg_dict = __read_to_dict(tar_file, dict_size)
|
||||
if reverse:
|
||||
src_dict = {v: k for k, v in src_dict.items()}
|
||||
trg_dict = {v: k for k, v in trg_dict.items()}
|
||||
return src_dict, trg_dict
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT14",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
paddle.dataset.common.download(URL_TRAIN, 'wmt14', MD5_TRAIN)
|
||||
paddle.dataset.common.download(URL_MODEL, 'wmt14', MD5_MODEL)
|
||||
@@ -0,0 +1,371 @@
|
||||
# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
|
||||
#
|
||||
# Licensed under the Apache License, Version 2.0 (the "License");
|
||||
# you may not use this file except in compliance with the License.
|
||||
# You may obtain a copy of the License at
|
||||
#
|
||||
# http://www.apache.org/licenses/LICENSE-2.0
|
||||
#
|
||||
# Unless required by applicable law or agreed to in writing, software
|
||||
# distributed under the License is distributed on an "AS IS" BASIS,
|
||||
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
|
||||
# See the License for the specific language governing permissions and
|
||||
# limitations under the License.
|
||||
"""
|
||||
ACL2016 Multimodal Machine Translation. Please see this website for more
|
||||
details: http://www.statmt.org/wmt16/multimodal-task.html#task1
|
||||
|
||||
If you use the dataset created for your task, please cite the following paper:
|
||||
Multi30K: Multilingual English-German Image Descriptions.
|
||||
|
||||
@article{elliott-EtAl:2016:VL16,
|
||||
author = {{Elliott}, D. and {Frank}, S. and {Sima"an}, K. and {Specia}, L.},
|
||||
title = {Multi30K: Multilingual English-German Image Descriptions},
|
||||
booktitle = {Proceedings of the 6th Workshop on Vision and Language},
|
||||
year = {2016},
|
||||
pages = {70--74},
|
||||
year = 2016
|
||||
}
|
||||
"""
|
||||
|
||||
import os
|
||||
import tarfile
|
||||
from collections import defaultdict
|
||||
|
||||
import paddle
|
||||
from paddle.utils import deprecated
|
||||
|
||||
__all__ = []
|
||||
|
||||
DATA_URL = "http://paddlemodels.bj.bcebos.com/wmt/wmt16.tar.gz"
|
||||
DATA_MD5 = "0c38be43600334966403524a40dcd81e"
|
||||
|
||||
TOTAL_EN_WORDS = 11250
|
||||
TOTAL_DE_WORDS = 19220
|
||||
|
||||
START_MARK = "<s>"
|
||||
END_MARK = "<e>"
|
||||
UNK_MARK = "<unk>"
|
||||
|
||||
|
||||
def __build_dict(tar_file, dict_size, save_path, lang):
|
||||
word_dict = defaultdict(int)
|
||||
with tarfile.open(tar_file, mode="r") as f:
|
||||
for line in f.extractfile("wmt16/train"):
|
||||
line = line.decode()
|
||||
line_split = line.strip().split("\t")
|
||||
if len(line_split) != 2:
|
||||
continue
|
||||
sen = line_split[0] if lang == "en" else line_split[1]
|
||||
for w in sen.split():
|
||||
word_dict[w] += 1
|
||||
|
||||
with open(save_path, "wb") as fout:
|
||||
fout.write((f"{START_MARK}\n{END_MARK}\n{UNK_MARK}\n").encode())
|
||||
for idx, word in enumerate(
|
||||
sorted(word_dict.items(), key=lambda x: x[1], reverse=True)
|
||||
):
|
||||
if idx + 3 == dict_size:
|
||||
break
|
||||
fout.write(word[0].encode())
|
||||
fout.write(b'\n')
|
||||
|
||||
|
||||
def __load_dict(tar_file, dict_size, lang, reverse=False):
|
||||
dict_path = os.path.join(
|
||||
paddle.dataset.common.DATA_HOME, f"wmt16/{lang}_{dict_size}.dict"
|
||||
)
|
||||
if not os.path.exists(dict_path) or (
|
||||
len(open(dict_path, "rb").readlines()) != dict_size
|
||||
):
|
||||
__build_dict(tar_file, dict_size, dict_path, lang)
|
||||
|
||||
word_dict = {}
|
||||
with open(dict_path, "rb") as fdict:
|
||||
for idx, line in enumerate(fdict):
|
||||
if reverse:
|
||||
word_dict[idx] = line.strip().decode()
|
||||
else:
|
||||
word_dict[line.strip().decode()] = idx
|
||||
return word_dict
|
||||
|
||||
|
||||
def __get_dict_size(src_dict_size, trg_dict_size, src_lang):
|
||||
src_dict_size = min(
|
||||
src_dict_size, (TOTAL_EN_WORDS if src_lang == "en" else TOTAL_DE_WORDS)
|
||||
)
|
||||
trg_dict_size = min(
|
||||
trg_dict_size, (TOTAL_DE_WORDS if src_lang == "en" else TOTAL_EN_WORDS)
|
||||
)
|
||||
return src_dict_size, trg_dict_size
|
||||
|
||||
|
||||
def reader_creator(tar_file, file_name, src_dict_size, trg_dict_size, src_lang):
|
||||
def reader():
|
||||
src_dict = __load_dict(tar_file, src_dict_size, src_lang)
|
||||
trg_dict = __load_dict(
|
||||
tar_file, trg_dict_size, ("de" if src_lang == "en" else "en")
|
||||
)
|
||||
|
||||
# the index for start mark, end mark, and unk are the same in source
|
||||
# language and target language. Here uses the source language
|
||||
# dictionary to determine their indices.
|
||||
start_id = src_dict[START_MARK]
|
||||
end_id = src_dict[END_MARK]
|
||||
unk_id = src_dict[UNK_MARK]
|
||||
|
||||
src_col = 0 if src_lang == "en" else 1
|
||||
trg_col = 1 - src_col
|
||||
|
||||
with tarfile.open(tar_file, mode="r") as f:
|
||||
for line in f.extractfile(file_name):
|
||||
line = line.decode()
|
||||
line_split = line.strip().split("\t")
|
||||
if len(line_split) != 2:
|
||||
continue
|
||||
src_words = line_split[src_col].split()
|
||||
src_ids = (
|
||||
[start_id]
|
||||
+ [src_dict.get(w, unk_id) for w in src_words]
|
||||
+ [end_id]
|
||||
)
|
||||
|
||||
trg_words = line_split[trg_col].split()
|
||||
trg_ids = [trg_dict.get(w, unk_id) for w in trg_words]
|
||||
|
||||
trg_ids_next = [*trg_ids, end_id]
|
||||
trg_ids = [start_id, *trg_ids]
|
||||
|
||||
yield src_ids, trg_ids, trg_ids_next
|
||||
|
||||
return reader
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT16",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def train(src_dict_size, trg_dict_size, src_lang="en"):
|
||||
"""
|
||||
WMT16 train set reader.
|
||||
|
||||
This function returns the reader for train data. Each sample the reader
|
||||
returns is made up of three fields: the source language word index sequence,
|
||||
target language word index sequence and next word index sequence.
|
||||
|
||||
|
||||
NOTE:
|
||||
The original like for training data is:
|
||||
http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/training.tar.gz
|
||||
|
||||
paddle.dataset.wmt16 provides a tokenized version of the original dataset by
|
||||
using moses's tokenization script:
|
||||
https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl
|
||||
|
||||
Args:
|
||||
src_dict_size(int): Size of the source language dictionary. Three
|
||||
special tokens will be added into the dictionary:
|
||||
<s> for start mark, <e> for end mark, and <unk> for
|
||||
unknown word.
|
||||
trg_dict_size(int): Size of the target language dictionary. Three
|
||||
special tokens will be added into the dictionary:
|
||||
<s> for start mark, <e> for end mark, and <unk> for
|
||||
unknown word.
|
||||
src_lang(string): A string indicating which language is the source
|
||||
language. Available options are: "en" for English
|
||||
and "de" for Germany.
|
||||
|
||||
Returns:
|
||||
callable: The train reader.
|
||||
"""
|
||||
|
||||
if src_lang not in ["en", "de"]:
|
||||
raise ValueError(
|
||||
"An error language type. Only support: "
|
||||
"en (for English); de(for Germany)."
|
||||
)
|
||||
src_dict_size, trg_dict_size = __get_dict_size(
|
||||
src_dict_size, trg_dict_size, src_lang
|
||||
)
|
||||
|
||||
return reader_creator(
|
||||
tar_file=paddle.dataset.common.download(
|
||||
DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"
|
||||
),
|
||||
file_name="wmt16/train",
|
||||
src_dict_size=src_dict_size,
|
||||
trg_dict_size=trg_dict_size,
|
||||
src_lang=src_lang,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT16",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def test(src_dict_size, trg_dict_size, src_lang="en"):
|
||||
"""
|
||||
WMT16 test set reader.
|
||||
|
||||
This function returns the reader for test data. Each sample the reader
|
||||
returns is made up of three fields: the source language word index sequence,
|
||||
target language word index sequence and next word index sequence.
|
||||
|
||||
NOTE:
|
||||
The original like for test data is:
|
||||
http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/mmt16_task1_test.tar.gz
|
||||
|
||||
paddle.dataset.wmt16 provides a tokenized version of the original dataset by
|
||||
using moses's tokenization script:
|
||||
https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl
|
||||
|
||||
Args:
|
||||
src_dict_size(int): Size of the source language dictionary. Three
|
||||
special tokens will be added into the dictionary:
|
||||
<s> for start mark, <e> for end mark, and <unk> for
|
||||
unknown word.
|
||||
trg_dict_size(int): Size of the target language dictionary. Three
|
||||
special tokens will be added into the dictionary:
|
||||
<s> for start mark, <e> for end mark, and <unk> for
|
||||
unknown word.
|
||||
src_lang(string): A string indicating which language is the source
|
||||
language. Available options are: "en" for English
|
||||
and "de" for Germany.
|
||||
|
||||
Returns:
|
||||
callable: The test reader.
|
||||
"""
|
||||
|
||||
if src_lang not in ["en", "de"]:
|
||||
raise ValueError(
|
||||
"An error language type. "
|
||||
"Only support: en (for English); de(for Germany)."
|
||||
)
|
||||
|
||||
src_dict_size, trg_dict_size = __get_dict_size(
|
||||
src_dict_size, trg_dict_size, src_lang
|
||||
)
|
||||
|
||||
return reader_creator(
|
||||
tar_file=paddle.dataset.common.download(
|
||||
DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"
|
||||
),
|
||||
file_name="wmt16/test",
|
||||
src_dict_size=src_dict_size,
|
||||
trg_dict_size=trg_dict_size,
|
||||
src_lang=src_lang,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT16",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def validation(src_dict_size, trg_dict_size, src_lang="en"):
|
||||
"""
|
||||
WMT16 validation set reader.
|
||||
|
||||
This function returns the reader for validation data. Each sample the reader
|
||||
returns is made up of three fields: the source language word index sequence,
|
||||
target language word index sequence and next word index sequence.
|
||||
|
||||
NOTE:
|
||||
The original like for validation data is:
|
||||
http://www.quest.dcs.shef.ac.uk/wmt16_files_mmt/validation.tar.gz
|
||||
|
||||
paddle.dataset.wmt16 provides a tokenized version of the original dataset by
|
||||
using moses's tokenization script:
|
||||
https://github.com/moses-smt/mosesdecoder/blob/master/scripts/tokenizer/tokenizer.perl
|
||||
|
||||
Args:
|
||||
src_dict_size(int): Size of the source language dictionary. Three
|
||||
special tokens will be added into the dictionary:
|
||||
<s> for start mark, <e> for end mark, and <unk> for
|
||||
unknown word.
|
||||
trg_dict_size(int): Size of the target language dictionary. Three
|
||||
special tokens will be added into the dictionary:
|
||||
<s> for start mark, <e> for end mark, and <unk> for
|
||||
unknown word.
|
||||
src_lang(string): A string indicating which language is the source
|
||||
language. Available options are: "en" for English
|
||||
and "de" for Germany.
|
||||
|
||||
Returns:
|
||||
callable: The validation reader.
|
||||
"""
|
||||
if src_lang not in ["en", "de"]:
|
||||
raise ValueError(
|
||||
"An error language type. "
|
||||
"Only support: en (for English); de(for Germany)."
|
||||
)
|
||||
src_dict_size, trg_dict_size = __get_dict_size(
|
||||
src_dict_size, trg_dict_size, src_lang
|
||||
)
|
||||
|
||||
return reader_creator(
|
||||
tar_file=paddle.dataset.common.download(
|
||||
DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"
|
||||
),
|
||||
file_name="wmt16/val",
|
||||
src_dict_size=src_dict_size,
|
||||
trg_dict_size=trg_dict_size,
|
||||
src_lang=src_lang,
|
||||
)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT16",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def get_dict(lang, dict_size, reverse=False):
|
||||
"""
|
||||
return the word dictionary for the specified language.
|
||||
|
||||
Args:
|
||||
lang(string): A string indicating which language is the source
|
||||
language. Available options are: "en" for English
|
||||
and "de" for Germany.
|
||||
dict_size(int): Size of the specified language dictionary.
|
||||
reverse(bool): If reverse is set to False, the returned python
|
||||
dictionary will use word as key and use index as value.
|
||||
If reverse is set to True, the returned python
|
||||
dictionary will use index as key and word as value.
|
||||
|
||||
Returns:
|
||||
dict: The word dictionary for the specific language.
|
||||
"""
|
||||
|
||||
if lang == "en":
|
||||
dict_size = min(dict_size, TOTAL_EN_WORDS)
|
||||
else:
|
||||
dict_size = min(dict_size, TOTAL_DE_WORDS)
|
||||
|
||||
dict_path = os.path.join(
|
||||
paddle.dataset.common.DATA_HOME, f"wmt16/{lang}_{dict_size}.dict"
|
||||
)
|
||||
assert os.path.exists(dict_path), "Word dictionary does not exist. "
|
||||
"Please invoke paddle.dataset.wmt16.train/test/validation first "
|
||||
"to build the dictionary."
|
||||
tar_file = os.path.join(paddle.dataset.common.DATA_HOME, "wmt16.tar.gz")
|
||||
return __load_dict(tar_file, dict_size, lang, reverse)
|
||||
|
||||
|
||||
@deprecated(
|
||||
since="2.0.0",
|
||||
update_to="paddle.text.datasets.WMT16",
|
||||
level=1,
|
||||
reason="Please use new dataset API which supports paddle.io.DataLoader",
|
||||
)
|
||||
def fetch():
|
||||
"""download the entire dataset."""
|
||||
paddle.v4.dataset.common.download(
|
||||
DATA_URL, "wmt16", DATA_MD5, "wmt16.tar.gz"
|
||||
)
|
||||
Reference in New Issue
Block a user