paddlepaddle--paddle/python/paddle/dataset/cifar.py

# Copyright (c) 2016 PaddlePaddle Authors. All Rights Reserved
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
#     http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
CIFAR dataset.

This module will download dataset from https://dataset.bj.bcebos.com/cifar/cifar-10-python.tar.gz and https://dataset.bj.bcebos.com/cifar/cifar-100-python.tar.gz, parse train/test set into
paddle reader creators.

The CIFAR-10 dataset consists of 60000 32x32 color images in 10 classes,
with 6000 images per class. There are 50000 training images and 10000 test
images.

The CIFAR-100 dataset is just like the CIFAR-10, except it has 100 classes
containing 600 images each. There are 500 training images and 100 testing
images per class.

"""

import os
import pickle
import tarfile

import numpy

import paddle.dataset.common
from paddle.utils import deprecated

__all__ = []

URL_PREFIX = 'https://dataset.bj.bcebos.com/cifar/'
CIFAR10_URL = URL_PREFIX + 'cifar-10-python.tar.gz'
CIFAR10_MD5 = 'c58f30108f718f92721af3b95e74349a'
CIFAR100_URL = URL_PREFIX + 'cifar-100-python.tar.gz'
CIFAR100_MD5 = 'eb9058c3a382ffc7106e4002c42a8d85'


def reader_creator(filename, sub_name, cycle=False, md5sum=None):
    def read_batch(batch):
        data = batch[b'data']
        labels = batch.get(b'labels', batch.get(b'fine_labels', None))
        assert labels is not None
        for sample, label in zip(data, labels):
            yield (sample / 255.0).astype(numpy.float32), int(label)

    verified_stat = None

    def reader():
        nonlocal verified_stat
        if md5sum is not None:
            stat = os.stat(filename)
            current_stat = (stat.st_mtime_ns, stat.st_size)
            if verified_stat != current_stat:
                file_md5 = paddle.dataset.common.md5file(filename)
                if file_md5 != md5sum:
                    raise ValueError(
                        "Loading unverified CIFAR pickle archive disabled. "
                        f"Please use the official MD5 {md5sum}."
                    )
                verified_stat = current_stat
        while True:
            with tarfile.open(filename, mode='r') as f:
                names = (
                    each_item.name
                    for each_item in f
                    if sub_name in each_item.name
                )

                for name in names:
                    batch = pickle.load(f.extractfile(name), encoding='bytes')
                    yield from read_batch(batch)

            if not cycle:
                break

    return reader


@deprecated(
    since="2.0.0",
    update_to="paddle.vision.datasets.Cifar100",
    level=1,
    reason="Please use new dataset API which supports paddle.io.DataLoader",
)
def train100():
    """
    CIFAR-100 training set creator.

    It returns a reader creator, each sample in the reader is image pixels in
    [0, 1] and label in [0, 99].

    :return: Training reader creator
    :rtype: callable
    """
    return reader_creator(
        paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5),
        'train',
        md5sum=CIFAR100_MD5,
    )


@deprecated(
    since="2.0.0",
    update_to="paddle.vision.datasets.Cifar100",
    level=1,
    reason="Please use new dataset API which supports paddle.io.DataLoader",
)
def test100():
    """
    CIFAR-100 test set creator.

    It returns a reader creator, each sample in the reader is image pixels in
    [0, 1] and label in [0, 99].

    :return: Test reader creator.
    :rtype: callable
    """
    return reader_creator(
        paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5),
        'test',
        md5sum=CIFAR100_MD5,
    )


@deprecated(
    since="2.0.0",
    update_to="paddle.vision.datasets.Cifar10",
    level=1,
    reason="Please use new dataset API which supports paddle.io.DataLoader",
)
def train10(cycle=False):
    """
    CIFAR-10 training set creator.

    It returns a reader creator, each sample in the reader is image pixels in
    [0, 1] and label in [0, 9].

    :param cycle: whether to cycle through the dataset
    :type cycle: bool
    :return: Training reader creator
    :rtype: callable
    """
    return reader_creator(
        paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5),
        'data_batch',
        cycle=cycle,
        md5sum=CIFAR10_MD5,
    )


@deprecated(
    since="2.0.0",
    update_to="paddle.vision.datasets.Cifar10",
    level=1,
    reason="Please use new dataset API which supports paddle.io.DataLoader",
)
def test10(cycle=False):
    """
    CIFAR-10 test set creator.

    It returns a reader creator, each sample in the reader is image pixels in
    [0, 1] and label in [0, 9].

    :param cycle: whether to cycle through the dataset
    :type cycle: bool
    :return: Test reader creator.
    :rtype: callable
    """
    return reader_creator(
        paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5),
        'test_batch',
        cycle=cycle,
        md5sum=CIFAR10_MD5,
    )


@deprecated(
    since="2.0.0",
    update_to="paddle.vision.datasets.Cifar10",
    level=1,
    reason="Please use new dataset API which supports paddle.io.DataLoader",
)
def fetch():
    paddle.dataset.common.download(CIFAR10_URL, 'cifar', CIFAR10_MD5)
    paddle.dataset.common.download(CIFAR100_URL, 'cifar', CIFAR100_MD5)