wehub-resource-sync
ee3943b5b1
CI / test (deps-minimum, windows-latest, unit) (push) Blocked by required conditions
CI / test_py314 (deps-latest, ubuntu-latest, unit) (push) Blocked by required conditions
CI / test_py314 (deps-latest, windows-latest, unit) (push) Blocked by required conditions
CI / test_py314_future (deps-latest, ubuntu-latest, unit) (push) Blocked by required conditions
CI / test_py314_future (deps-latest, windows-latest, unit) (push) Blocked by required conditions
Secret Leaks / trufflehog (push) Waiting to run
CI / check_code_quality (push) Waiting to run
CI / test (deps-latest, ubuntu-latest, integration) (push) Blocked by required conditions
CI / test (deps-latest, ubuntu-latest, unit) (push) Blocked by required conditions
CI / test (deps-latest, windows-latest, integration) (push) Blocked by required conditions
CI / test (deps-latest, windows-latest, unit) (push) Blocked by required conditions
CI / test (deps-minimum, ubuntu-latest, integration) (push) Blocked by required conditions
CI / test (deps-minimum, ubuntu-latest, unit) (push) Blocked by required conditions
CI / test (deps-minimum, windows-latest, integration) (push) Blocked by required conditions
Build documentation / build (push) Failing after 0s
42 lines
1.6 KiB
Python
42 lines
1.6 KiB
Python
import pytest
|
|
|
|
from datasets import load_dataset
|
|
from datasets.builder import InvalidConfigName
|
|
from datasets.data_files import DataFilesList
|
|
from datasets.packaged_modules.parquet.parquet import ParquetConfig
|
|
|
|
|
|
def test_config_raises_when_invalid_name() -> None:
|
|
with pytest.raises(InvalidConfigName, match="Bad characters"):
|
|
_ = ParquetConfig(name="name-with-*-invalid-character")
|
|
|
|
|
|
@pytest.mark.parametrize("data_files", ["str_path", ["str_path"], DataFilesList(["str_path"], [()])])
|
|
def test_config_raises_when_invalid_data_files(data_files) -> None:
|
|
with pytest.raises(ValueError, match="Expected a DataFilesDict"):
|
|
_ = ParquetConfig(name="name", data_files=data_files)
|
|
|
|
|
|
def test_parquet_reshard(multi_row_groups_parquet_path):
|
|
ds = load_dataset("parquet", data_files=multi_row_groups_parquet_path, split="train", streaming=True)
|
|
assert ds.num_shards == 1
|
|
expected = list(ds)
|
|
resharded_ds = ds.reshard()
|
|
assert resharded_ds.num_shards == 4
|
|
assert list(resharded_ds) == expected
|
|
|
|
|
|
def test_parquet_columns(parquet_path):
|
|
ds = load_dataset("parquet", data_files=parquet_path, split="train", streaming=True)
|
|
full_features = ds.features
|
|
assert len(ds.features) == 3
|
|
assert len(next(iter(ds))) == 3
|
|
ds = load_dataset("parquet", data_files=parquet_path, split="train", streaming=True, columns=["col_1"])
|
|
assert len(ds.features) == 1
|
|
assert len(next(iter(ds))) == 1
|
|
ds = load_dataset(
|
|
"parquet", data_files=parquet_path, split="train", streaming=True, columns=["col_1"], features=full_features
|
|
)
|
|
assert len(ds.features) == 1
|
|
assert len(next(iter(ds))) == 1
|