chore: import upstream snapshot with attribution

.dvc/.gitignore

@@ -0,0 +1,3 @@
+/config.local
+/tmp
+/cache

.dvc/plots/confusion.json

@@ -0,0 +1,30 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": "rect",
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "nominal",
+            "sort": "ascending",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "nominal",
+            "sort": "ascending",
+            "title": "<DVC_METRIC_Y_LABEL>"
+        },
+        "color": {
+            "aggregate": "count",
+            "type": "quantitative"
+        },
+        "facet": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    }
+}

.dvc/plots/default.json

@@ -0,0 +1,29 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": {
+        "type": "line"
+    },
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_Y_LABEL>",
+            "scale": {
+                "zero": false
+            }
+        },
+        "color": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    }
+}

.dvc/plots/scatter.json

@@ -0,0 +1,27 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": "point",
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_Y_LABEL>",
+            "scale": {
+                "zero": false
+            }
+        },
+        "color": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    }
+}

.dvc/plots/smooth.json

@@ -0,0 +1,39 @@
+{
+    "$schema": "https://vega.github.io/schema/vega-lite/v4.json",
+    "data": {
+        "values": "<DVC_METRIC_DATA>"
+    },
+    "title": "<DVC_METRIC_TITLE>",
+    "mark": {
+        "type": "line"
+    },
+    "encoding": {
+        "x": {
+            "field": "<DVC_METRIC_X>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_X_LABEL>"
+        },
+        "y": {
+            "field": "<DVC_METRIC_Y>",
+            "type": "quantitative",
+            "title": "<DVC_METRIC_Y_LABEL>",
+            "scale": {
+                "zero": false
+            }
+        },
+        "color": {
+            "field": "rev",
+            "type": "nominal"
+        }
+    },
+    "transform": [
+        {
+            "loess": "<DVC_METRIC_Y>",
+            "on": "<DVC_METRIC_X>",
+            "groupby": [
+                "rev"
+            ],
+            "bandwidth": 0.3
+        }
+    ]
+}

.dvcignore

@@ -0,0 +1,3 @@
+# Add patterns of files dvc should ignore, which could improve
+# the performance. Learn more at
+# https://dvc.org/doc/user-guide/dvcignore

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,45 @@
+name: Bug report
+description: Create a report to help reproduce and fix the bug
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce the bug
+      description: |
+        Please provide a code sample that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
+        If you have code snippets, error messages, stack traces please provide them here as well.
+        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
+        Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
+      placeholder: |
+        Steps to reproduce the behavior:
+          
+          1.
+          2.
+          3.
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of the expected results.
+
+  - type: textarea
+    id: environment-info
+    attributes:
+      label: Environment info
+      description: Please share your environemnt info with us. You can run the command `datasets-cli env` and copy-paste its output below.
+      placeholder: datasets version, platform, python version, ...
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+contact_links:
+  - name: Datasets on the Hugging Face Hub
+    url: https://huggingface.co/datasets
+    about: Please use the "Community" tab of the dataset on the Hugging Face Hub to open a discussion or a pull request
+  - name: Forum
+    url: https://discuss.huggingface.co/c/datasets/10
+    about: Please ask and answer questions here, and engage with other community members

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,29 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: feature-request
+    attributes:
+      label: Feature request
+      description: A clear and concise description of the feature proposal.
+    validations:
+      required: true
+  
+  - type: textarea
+    id: motivation
+    validations:
+      required: true
+    attributes:
+      label: Motivation
+      description: |
+        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.   
+
+  - type: textarea
+    id: contribution
+    validations:
+      required: true
+    attributes:
+      label: Your contribution
+      description: |
+        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md).

.github/conda/build.sh

@@ -0,0 +1 @@
+$PYTHON setup.py install --single-version-externally-managed --record=record.txt

.github/conda/meta.yaml

@@ -0,0 +1,55 @@
+{% set name = "datasets" %}
+
+package:
+  name: "{{ name|lower }}"
+  version: "{{ DATASETS_VERSION }}"
+
+source:
+  path: ../../
+
+build:
+  noarch: python
+
+requirements:
+  host:
+    - python
+    - pip
+    - numpy >=1.17
+    - pyarrow >=16.0.0
+    - python-xxhash
+    - dill
+    - pandas
+    - requests >=2.19.0
+    - httpx <1.0.0
+    - tqdm >=4.66.3
+    - dataclasses
+    - multiprocess
+    - fsspec
+    - huggingface_hub >=0.25.0,<2.0.0
+    - packaging
+  run:
+    - python
+    - pip
+    - numpy >=1.17
+    - pyarrow >=16.0.0
+    - python-xxhash
+    - dill
+    - pandas
+    - requests >=2.19.0
+    - httpx <1.0.0
+    - tqdm >=4.66.3
+    - dataclasses
+    - multiprocess
+    - fsspec
+    - huggingface_hub >=0.25.0,<2.0.0
+    - packaging
+
+test:
+  imports:
+    - datasets
+
+about:
+  home: https://huggingface.co
+  license: Apache License 2.0
+  license_file: LICENSE
+  summary: "🤗 The largest hub of ready-to-use NLP datasets for ML models with fast, easy-to-use and efficient data manipulation tools"

.github/workflows/build_documentation.yml

@@ -0,0 +1,20 @@
+name: Build documentation
+
+on:
+  push:
+    branches:
+      - main
+      - doc-builder*
+      - v*-release
+      - v*-patch
+
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
+    with:
+      commit_sha: ${{ github.sha }}
+      package: datasets
+      notebook_folder: datasets_doc
+    secrets:
+      token: ${{ secrets.HUGGINGFACE_PUSH }}
+      hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}

.github/workflows/build_pr_documentation.yml

@@ -0,0 +1,16 @@
+name: Build PR Documentation
+
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
+    with:
+      commit_sha: ${{ github.event.pull_request.head.sha }}
+      pr_number: ${{ github.event.number }}
+      package: datasets

.github/workflows/ci.yml

@@ -0,0 +1,176 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+      - ci-*
+
+env:
+  CI_HEADERS: ${{ secrets.CI_HEADERS }}
+
+jobs:
+
+  check_code_quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[quality]
+      - name: Check quality
+        run: |
+          ruff check tests src benchmarks utils setup.py # linter
+          ruff format --check tests src benchmarks utils setup.py # formatter
+
+  test:
+    needs: check_code_quality
+    strategy:
+      matrix:
+        test: ['unit', 'integration']
+        os: [ubuntu-latest, windows-latest]
+        deps_versions: [deps-latest, deps-minimum]
+    continue-on-error: ${{ matrix.test == 'integration' }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+      - name: Setup FFmpeg
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg 
+      - name: Set up Python 3.10
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.10"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.10"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Install uv
+        run: pip install --upgrade uv
+      - name: Install dependencies
+        run: uv pip install --system "datasets[tests] @ ."
+      - name: Install dependencies (latest versions)
+        if: ${{ matrix.deps_versions == 'deps-latest' }}
+        run: uv pip install --system --upgrade pyarrow huggingface-hub "dill<0.3.9"
+      - name: Install dependencies (minimum versions)
+        if: ${{ matrix.deps_versions != 'deps-latest' }}
+        run: uv pip install --system pyarrow==21.0.0 huggingface-hub==0.25.0 transformers dill==0.3.1.1
+      - name: Print dependencies
+        run: uv pip list
+      - name: Test with pytest
+        run: |
+          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/
+
+  test_py314:
+    needs: check_code_quality
+    strategy:
+      matrix:
+        test: ['unit']
+        os: [ubuntu-latest, windows-latest]
+        deps_versions: [deps-latest]
+    continue-on-error: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+      - name: Setup FFmpeg
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.14"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.14"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Install uv
+        run: pip install --upgrade uv
+      - name: Install dependencies
+        run: uv pip install --system "datasets[tests] @ ."
+      - name: Install tsfile (py3.14 only)
+        run: uv pip install --system "tsfile>=2.3.0"
+      - name: Print dependencies
+        run: uv pip list
+      - name: Test with pytest
+        run: |
+          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/
+
+  test_py314_future:
+    needs: check_code_quality
+    strategy:
+      matrix:
+        test: ['unit']
+        os: [ubuntu-latest, windows-latest]
+        deps_versions: [deps-latest]
+    continue-on-error: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+      - name: Setup FFmpeg
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg 
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.14"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.14"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Install uv
+        run: pip install --upgrade uv
+      - name: Install dependencies
+        run: uv pip install --system "datasets[tests_numpy2] @ ."
+      - name: Print dependencies
+        run: pip list
+
+      - name: Test with pytest
+        run: |
+          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/

.github/workflows/release-conda.yml

@@ -0,0 +1,47 @@
+name: Release - Conda
+
+on:
+  push:
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]+*"
+
+env:
+  ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+
+jobs:
+  build_and_package:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install miniconda
+        uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5  # v4.0.1
+        with:
+          auto-update-conda: true
+          auto-activate-base: false
+          python-version: "3.10"
+          channels: huggingface
+
+      - name: Setup conda env
+        run: |
+          # Install conda-build into base so the command is available
+          conda install -y -c defaults -c conda-forge anaconda-client conda-build
+          # Create the build environment
+          conda create -y -n build-datasets python=3.10
+          conda activate build-datasets
+
+      - name: Extract version
+        run: echo "DATASETS_VERSION=`python setup.py --version`" >> $GITHUB_ENV
+
+      - name: Build conda packages
+        run: |
+          conda-build --override-channels -c conda-forge -c huggingface .github/conda
+
+      - name: Upload to Anaconda
+        run: |
+          anaconda upload `conda-build .github/conda --output -c conda-forge` --force

.github/workflows/self-assign.yaml

@@ -0,0 +1,16 @@
+name: Self-assign
+on:
+  issue_comment:
+    types: created
+jobs:
+  one:
+    runs-on: ubuntu-latest
+    if: >-
+      (github.event.comment.body == '#take' ||
+       github.event.comment.body == '#self-assign')
+      && !github.event.issue.assignee
+    steps:
+      - run: |
+          echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -X "DELETE" https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels/help%20wanted

.github/workflows/trufflehog.yml

@@ -0,0 +1,20 @@
+on:
+  push:
+
+name: Secret Leaks
+
+permissions:
+  contents: read
+
+jobs:
+  trufflehog:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      with:
+        fetch-depth: 0
+    - name: Secret Scanning
+      uses: trufflesecurity/trufflehog@6bd2d14f7a4bc1e569fa3550efa7ec632a4fa67b  # main
+      with:
+        extra_args: --results=verified

.github/workflows/upload_pr_documentation.yml

@@ -0,0 +1,16 @@
+name: Upload PR Documentation
+
+on:
+  workflow_run:
+    workflows: ["Build PR Documentation"]
+    types:
+      - completed
+
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@9ad2de8582b56c017cb530c1165116d40433f1c6  # main
+    with:
+      package_name: datasets
+    secrets:
+      hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
+      comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}

.gitignore

@@ -0,0 +1,60 @@
+# Locked files
+*.lock
+!dvc.lock
+
+# Extracted dummy data
+datasets/**/dummy_data-zip-extracted/
+
+# Compiled python modules.
+*.pyc
+
+# Byte-compiled
+_pycache__/
+.cache/
+
+# Python egg metadata, regenerated from source files by setuptools.
+*.egg-info
+.eggs/
+
+# PyPI distribution artifacts.
+build/
+dist/
+
+# Environments
+.env
+.venv
+env/
+venv/
+ENV/
+env.bak/
+venv.bak/
+
+# pyenv
+.python-version
+
+# Tests
+.pytest_cache/
+
+# Other
+*.DS_Store
+
+# PyCharm/vscode
+.idea
+.vscode
+
+# Vim
+.*.swp
+
+# playground
+/playground
+
+# Sphinx documentation
+docs/_build/
+docs/source/_build/
+
+# Benchmark results
+report.json
+report.md
+
+# Ruff
+.ruff_cache

.pre-commit-config.yaml

@@ -0,0 +1,9 @@
+repos:
+  - repo: https://github.com/charliermarsh/ruff-pre-commit # https://github.com/charliermarsh/ruff#usage
+    rev: 'v0.11.8'
+    hooks:
+      # Run the linter.
+      - id: ruff
+        args: [ --fix ]
+      # Run the formatter.
+      - id: ruff-format

.zenodo.json

@@ -0,0 +1,130 @@
+{
+    "license": "Apache-2.0",
+    "creators": [
+        {
+            "affiliation": "Hugging Face",
+            "name": "Quentin Lhoest"
+        },
+        {
+            "orcid": "0000-0003-1727-1045",
+            "affiliation": "Hugging Face",
+            "name": "Albert Villanova del Moral"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Patrick von Platen"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Thomas Wolf"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Mario Šaško"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Yacine Jernite"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Abhishek Thakur"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Lewis Tunstall"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Suraj Patil"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Mariama Drame"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Julien Chaumond"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Julien Plu"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Joe Davison"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Simon Brandeis"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Victor Sanh"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Teven Le Scao"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Kevin Canwen Xu"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Nicolas Patry"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Steven Liu"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Angelina McMillan-Major"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Philipp Schmid"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Sylvain Gugger"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Nathan Raw"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Sylvain Lesage"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Anton Lozhkov"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Matthew Carrigan"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Th\u00e9o Matussi\u00e8re"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Leandro von Werra"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Lysandre Debut"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Stas Bekman"
+        },
+        {
+            "affiliation": "Hugging Face",
+            "name": "Cl\u00e9ment Delangue"
+        }
+    ]
+}

ADD_NEW_DATASET.md

@@ -0,0 +1,8 @@
+# How to add one new datasets
+
+Add datasets directly to the 🤗 Hugging Face Hub!
+
+You can share your dataset on https://huggingface.co/datasets directly using your account, see the documentation:
+
+* [Create a dataset and upload files on the website](https://huggingface.co/docs/datasets/upload_dataset)
+* [Advanced guide using the CLI](https://huggingface.co/docs/datasets/share)

AUTHORS

@@ -0,0 +1,8 @@
+# This is the list of HuggingFace Datasets authors for copyright purposes.
+#
+# This does not necessarily list everyone who has contributed code, since in
+# some cases, their employer may be the copyright holder.  To see the full list
+# of contributors, see the revision history in source control.
+
+Google Inc.
+HuggingFace Inc.

CITATION.cff

@@ -0,0 +1,150 @@
+cff-version: 1.2.0
+message: "If you use this software, please cite it as below."
+title: "huggingface/datasets"
+authors:
+- family-names: Lhoest
+  given-names: Quentin
+- family-names: Villanova del Moral
+  given-names: Albert
+  orcid: "https://orcid.org/0000-0003-1727-1045"
+- family-names: von Platen
+  given-names: Patrick
+- family-names: Wolf
+  given-names: Thomas
+- family-names: Šaško
+  given-names: Mario
+- family-names: Jernite
+  given-names: Yacine
+- family-names: Thakur
+  given-names: Abhishek
+- family-names: Tunstall
+  given-names: Lewis
+- family-names: Patil
+  given-names: Suraj
+- family-names: Drame
+  given-names: Mariama
+- family-names: Chaumond
+  given-names: Julien
+- family-names: Plu
+  given-names: Julien
+- family-names: Davison
+  given-names: Joe
+- family-names: Brandeis
+  given-names: Simon
+- family-names: Sanh
+  given-names: Victor
+- family-names: Le Scao
+  given-names: Teven
+- family-names: Canwen Xu
+  given-names: Kevin
+- family-names: Patry
+  given-names: Nicolas
+- family-names: Liu
+  given-names: Steven
+- family-names: McMillan-Major
+  given-names: Angelina
+- family-names: Schmid
+  given-names: Philipp
+- family-names: Gugger
+  given-names: Sylvain
+- family-names: Raw
+  given-names: Nathan
+- family-names: Lesage
+  given-names: Sylvain
+- family-names: Lozhkov
+  given-names: Anton
+- family-names: Carrigan
+  given-names: Matthew
+- family-names: Matussière
+  given-names: Théo
+- family-names: von Werra
+  given-names: Leandro
+- family-names: Debut
+  given-names: Lysandre
+- family-names: Bekman
+  given-names: Stas
+- family-names: Delangue
+  given-names: Clément
+doi: 10.5281/zenodo.4817768
+repository-code: "https://github.com/huggingface/datasets"
+license: Apache-2.0
+preferred-citation:
+  type: conference-paper
+  title: "Datasets: A Community Library for Natural Language Processing"
+  authors:
+  - family-names: Lhoest
+    given-names: Quentin
+  - family-names: Villanova del Moral
+    given-names: Albert
+    orcid: "https://orcid.org/0000-0003-1727-1045"
+  - family-names: von Platen
+    given-names: Patrick
+  - family-names: Wolf
+    given-names: Thomas
+  - family-names: Šaško
+    given-names: Mario
+  - family-names: Jernite
+    given-names: Yacine
+  - family-names: Thakur
+    given-names: Abhishek
+  - family-names: Tunstall
+    given-names: Lewis
+  - family-names: Patil
+    given-names: Suraj
+  - family-names: Drame
+    given-names: Mariama
+  - family-names: Chaumond
+    given-names: Julien
+  - family-names: Plu
+    given-names: Julien
+  - family-names: Davison
+    given-names: Joe
+  - family-names: Brandeis
+    given-names: Simon
+  - family-names: Sanh
+    given-names: Victor
+  - family-names: Le Scao
+    given-names: Teven
+  - family-names: Canwen Xu
+    given-names: Kevin
+  - family-names: Patry
+    given-names: Nicolas
+  - family-names: Liu
+    given-names: Steven
+  - family-names: McMillan-Major
+    given-names: Angelina
+  - family-names: Schmid
+    given-names: Philipp
+  - family-names: Gugger
+    given-names: Sylvain
+  - family-names: Raw
+    given-names: Nathan
+  - family-names: Lesage
+    given-names: Sylvain
+  - family-names: Lozhkov
+    given-names: Anton
+  - family-names: Carrigan
+    given-names: Matthew
+  - family-names: Matussière
+    given-names: Théo
+  - family-names: von Werra
+    given-names: Leandro
+  - family-names: Debut
+    given-names: Lysandre
+  - family-names: Bekman
+    given-names: Stas
+  - family-names: Delangue
+    given-names: Clément
+  collection-title: "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations"
+  collection-type: proceedings
+  month: 11
+  year: 2021
+  publisher:
+    name: "Association for Computational Linguistics"
+  url: "https://aclanthology.org/2021.emnlp-demo.21"
+  start: 175
+  end: 184
+  identifiers:
+    - type: other
+      value: "arXiv:2109.02846"
+      description: "The arXiv preprint of the paper"

CODE_OF_CONDUCT.md

@@ -0,0 +1,132 @@
+# Contributor Covenant Code of Conduct
+
+## Our Pledge
+
+We as members, contributors, and leaders pledge to make participation in our
+community a harassment-free experience for everyone, regardless of age, body
+size, visible or invisible disability, ethnicity, sex characteristics, gender
+identity and expression, level of experience, education, socio-economic status,
+nationality, personal appearance, race, caste, color, religion, or sexual identity
+and orientation.
+
+We pledge to act and interact in ways that contribute to an open, welcoming,
+diverse, inclusive, and healthy community.
+
+## Our Standards
+
+Examples of behavior that contributes to a positive environment for our
+community include:
+
+* Demonstrating empathy and kindness toward other people
+* Being respectful of differing opinions, viewpoints, and experiences
+* Giving and gracefully accepting constructive feedback
+* Accepting responsibility and apologizing to those affected by our mistakes,
+  and learning from the experience
+* Focusing on what is best not just for us as individuals, but for the
+  overall community
+
+Examples of unacceptable behavior include:
+
+* The use of sexualized language or imagery, and sexual attention or
+  advances of any kind
+* Trolling, insulting or derogatory comments, and personal or political attacks
+* Public or private harassment
+* Publishing others' private information, such as a physical or email
+  address, without their explicit permission
+* Other conduct which could reasonably be considered inappropriate in a
+  professional setting
+
+## Enforcement Responsibilities
+
+Community leaders are responsible for clarifying and enforcing our standards of
+acceptable behavior and will take appropriate and fair corrective action in
+response to any behavior that they deem inappropriate, threatening, offensive,
+or harmful.
+
+Community leaders have the right and responsibility to remove, edit, or reject
+comments, commits, code, wiki edits, issues, and other contributions that are
+not aligned to this Code of Conduct, and will communicate reasons for moderation
+decisions when appropriate.
+
+## Scope
+
+This Code of Conduct applies within all community spaces, and also applies when
+an individual is officially representing the community in public spaces.
+Examples of representing our community include using an official e-mail address,
+posting via an official social media account, or acting as an appointed
+representative at an online or offline event.
+
+## Enforcement
+
+Instances of abusive, harassing, or otherwise unacceptable behavior may be
+reported to the community leaders responsible for enforcement at
+feedback@huggingface.co.
+All complaints will be reviewed and investigated promptly and fairly.
+
+All community leaders are obligated to respect the privacy and security of the
+reporter of any incident.
+
+## Enforcement Guidelines
+
+Community leaders will follow these Community Impact Guidelines in determining
+the consequences for any action they deem in violation of this Code of Conduct:
+
+### 1. Correction
+
+**Community Impact**: Use of inappropriate language or other behavior deemed
+unprofessional or unwelcome in the community.
+
+**Consequence**: A private, written warning from community leaders, providing
+clarity around the nature of the violation and an explanation of why the
+behavior was inappropriate. A public apology may be requested.
+
+### 2. Warning
+
+**Community Impact**: A violation through a single incident or series
+of actions.
+
+**Consequence**: A warning with consequences for continued behavior. No
+interaction with the people involved, including unsolicited interaction with
+those enforcing the Code of Conduct, for a specified period of time. This
+includes avoiding interactions in community spaces as well as external channels
+like social media. Violating these terms may lead to a temporary or
+permanent ban.
+
+### 3. Temporary Ban
+
+**Community Impact**: A serious violation of community standards, including
+sustained inappropriate behavior.
+
+**Consequence**: A temporary ban from any sort of interaction or public
+communication with the community for a specified period of time. No public or
+private interaction with the people involved, including unsolicited interaction
+with those enforcing the Code of Conduct, is allowed during this period.
+Violating these terms may lead to a permanent ban.
+
+### 4. Permanent Ban
+
+**Community Impact**: Demonstrating a pattern of violation of community
+standards, including sustained inappropriate behavior,  harassment of an
+individual, or aggression toward or disparagement of classes of individuals.
+
+**Consequence**: A permanent ban from any sort of public interaction within
+the community.
+
+## Attribution
+
+This Code of Conduct is adapted from the [Contributor Covenant][homepage],
+version 2.0, available at
+[https://www.contributor-covenant.org/version/2/0/code_of_conduct.html][v2.0].
+
+Community Impact Guidelines were inspired by 
+[Mozilla's code of conduct enforcement ladder][Mozilla CoC].
+
+For answers to common questions about this code of conduct, see the FAQ at
+[https://www.contributor-covenant.org/faq][FAQ]. Translations are available 
+at [https://www.contributor-covenant.org/translations][translations].
+
+[homepage]: https://www.contributor-covenant.org
+[v2.0]: https://www.contributor-covenant.org/version/2/0/code_of_conduct.html
+[Mozilla CoC]: https://github.com/mozilla/diversity
+[FAQ]: https://www.contributor-covenant.org/faq
+[translations]: https://www.contributor-covenant.org/translations

CONTRIBUTING.md

@@ -0,0 +1,130 @@
+# How to contribute to Datasets?
+[![Contributor Covenant](https://img.shields.io/badge/Contributor%20Covenant-2.0-4baaaa.svg)](CODE_OF_CONDUCT.md)
+
+Datasets is an open source project, so all contributions and suggestions are welcome.
+
+You can contribute in many different ways: giving ideas, answering questions, reporting bugs, proposing enhancements,
+improving the documentation, fixing bugs,...
+
+Many thanks in advance to every contributor.
+
+In order to facilitate healthy, constructive behavior in an open and inclusive community, we all respect and abide by
+our [code of conduct](CODE_OF_CONDUCT.md).
+
+## How to work on an open Issue?
+You have the list of open Issues at: https://github.com/huggingface/datasets/issues
+
+Some of them may have the label `help wanted`: that means that any contributor is welcomed!
+
+If you would like to work on any of the open Issues:
+
+1. Make sure it is not already assigned to someone else. You have the assignee (if any) on the top of the right column of the Issue page.
+
+2. You can self-assign it by commenting on the Issue page with the keyword: `#self-assign`.
+
+3. Work on your self-assigned issue and eventually create a Pull Request.
+
+## How to create a Pull Request?
+If you want to add a dataset see specific instructions in the section [*How to add a dataset*](#how-to-add-a-dataset).
+
+1. Fork the [repository](https://github.com/huggingface/datasets) by clicking on the 'Fork' button on the repository's page. This creates a copy of the code under your GitHub user account.
+
+2. Clone your fork to your local disk, and add the base repository as a remote:
+
+    ```bash
+    git clone git@github.com:<your Github handle>/datasets.git
+    cd datasets
+    git remote add upstream https://github.com/huggingface/datasets.git
+    ```
+
+3. Create a new branch to hold your development changes:
+
+    ```bash
+    git checkout -b a-descriptive-name-for-my-changes
+    ```
+
+    **do not** work on the `main` branch.
+
+4. Set up a development environment by running the following command in a virtual environment:
+
+    Simple setup with code formatting only (recommended)
+    ```bash
+    pip install -e ".[quality]"
+    ```
+    
+    Advanced setup with all the optional dependencies
+    ```bash
+    pip install -e ".[dev]"
+    ```
+
+   (If datasets was already installed in the virtual environment, remove
+   it with `pip uninstall datasets` before reinstalling it in editable
+   mode with the `-e` flag.)
+
+5. Develop the features on your branch.
+
+6. Format your code. Run `black` and `ruff` so that your newly added files look nice with the following command:
+
+    ```bash
+    make style
+    ```
+   
+7. _(Optional)_ You can also use [`pre-commit`](https://pre-commit.com/) to format your code automatically each time run `git commit`, instead of running `make style` manually. 
+To do this, install `pre-commit` via `pip install pre-commit` and then run `pre-commit install` in the project's root directory to set up the hooks.
+Note that if any files were formatted by `pre-commit` hooks during committing, you have to run `git commit` again .
+
+
+8. Once you're happy with your contribution, add your changed files and make a commit to record your changes locally:
+
+    ```bash
+    git add -u
+    git commit
+    ```
+
+    It is a good idea to sync your copy of the code with the original
+    repository regularly. This way you can quickly account for changes:
+
+    ```bash
+    git fetch upstream
+    git rebase upstream/main
+    ```
+
+9. Once you are satisfied, push the changes to your fork repo using:
+
+   ```bash
+   git push -u origin a-descriptive-name-for-my-changes
+   ```
+
+   Go the webpage of your fork on GitHub. Click on "Pull request" to send your changes to the project maintainers for review.
+
+## Datasets on Hugging Face
+
+### How to add a dataset on Hugging Face
+
+You can share your dataset on https://huggingface.co/datasets directly using your account (no need to open a PR on GitHub), see the documentation:
+
+* [Create a dataset and upload files on the website](https://huggingface.co/docs/datasets/upload_dataset)
+* [Advanced guide using the CLI](https://huggingface.co/docs/datasets/share)
+
+### How to contribute to the dataset cards
+
+Improving the documentation of datasets is an ever-increasing effort, and we invite users to contribute by sharing their insights with the community in the `README.md` dataset cards provided for each dataset.
+
+If you see that a dataset card is missing information that you are in a position to provide (as an author of the dataset or as an experienced user), the best thing you can do is to open a Pull Request on the Hugging Face Hub. To do, go to the "Files and versions" tab of the dataset page and edit the `README.md` file. We provide:
+
+* a [template](https://github.com/huggingface/datasets/blob/main/templates/README.md)
+* a [guide](https://github.com/huggingface/datasets/blob/main/templates/README_guide.md) describing what information should go into each of the paragraphs
+* and if you need inspiration, we recommend looking through a [completed example](https://huggingface.co/datasets/eli5/blob/main/README.md)
+
+If you are a **dataset author**... you know what to do, it is your dataset after all ;) ! We would especially appreciate if you could help us fill in information about the process of creating the dataset, and take a moment to reflect on its social impact and possible limitations if you haven't already done so in the dataset paper or in another data statement.
+
+If you are a **user of a dataset**, the main source of information should be the dataset paper if it is available: we recommend pulling information from there into the relevant paragraphs of the template. We also eagerly welcome discussions on the [Considerations for Using the Data](https://github.com/huggingface/datasets/blob/main/templates/README_guide.md#considerations-for-using-the-data) based on existing scholarship or personal experience that would benefit the whole community.
+
+Finally, if you want more information on the how and why of dataset cards, we strongly recommend reading the foundational works [Datasheets for Datasets](https://huggingface.co/papers/1803.09010) and [Data Statements for NLP](https://www.aclweb.org/anthology/Q18-1041/).
+
+Thank you for your contribution!
+
+## Code of conduct
+
+This project adheres to the HuggingFace [code of conduct](CODE_OF_CONDUCT.md).
+By participating, you are expected to abide by this code.

LICENSE

@@ -0,0 +1,202 @@
+
+                                 Apache License
+                           Version 2.0, January 2004
+                        http://www.apache.org/licenses/
+
+   TERMS AND CONDITIONS FOR USE, REPRODUCTION, AND DISTRIBUTION
+
+   1. Definitions.
+
+      "License" shall mean the terms and conditions for use, reproduction,
+      and distribution as defined by Sections 1 through 9 of this document.
+
+      "Licensor" shall mean the copyright owner or entity authorized by
+      the copyright owner that is granting the License.
+
+      "Legal Entity" shall mean the union of the acting entity and all
+      other entities that control, are controlled by, or are under common
+      control with that entity. For the purposes of this definition,
+      "control" means (i) the power, direct or indirect, to cause the
+      direction or management of such entity, whether by contract or
+      otherwise, or (ii) ownership of fifty percent (50%) or more of the
+      outstanding shares, or (iii) beneficial ownership of such entity.
+
+      "You" (or "Your") shall mean an individual or Legal Entity
+      exercising permissions granted by this License.
+
+      "Source" form shall mean the preferred form for making modifications,
+      including but not limited to software source code, documentation
+      source, and configuration files.
+
+      "Object" form shall mean any form resulting from mechanical
+      transformation or translation of a Source form, including but
+      not limited to compiled object code, generated documentation,
+      and conversions to other media types.
+
+      "Work" shall mean the work of authorship, whether in Source or
+      Object form, made available under the License, as indicated by a
+      copyright notice that is included in or attached to the work
+      (an example is provided in the Appendix below).
+
+      "Derivative Works" shall mean any work, whether in Source or Object
+      form, that is based on (or derived from) the Work and for which the
+      editorial revisions, annotations, elaborations, or other modifications
+      represent, as a whole, an original work of authorship. For the purposes
+      of this License, Derivative Works shall not include works that remain
+      separable from, or merely link (or bind by name) to the interfaces of,
+      the Work and Derivative Works thereof.
+
+      "Contribution" shall mean any work of authorship, including
+      the original version of the Work and any modifications or additions
+      to that Work or Derivative Works thereof, that is intentionally
+      submitted to Licensor for inclusion in the Work by the copyright owner
+      or by an individual or Legal Entity authorized to submit on behalf of
+      the copyright owner. For the purposes of this definition, "submitted"
+      means any form of electronic, verbal, or written communication sent
+      to the Licensor or its representatives, including but not limited to
+      communication on electronic mailing lists, source code control systems,
+      and issue tracking systems that are managed by, or on behalf of, the
+      Licensor for the purpose of discussing and improving the Work, but
+      excluding communication that is conspicuously marked or otherwise
+      designated in writing by the copyright owner as "Not a Contribution."
+
+      "Contributor" shall mean Licensor and any individual or Legal Entity
+      on behalf of whom a Contribution has been received by Licensor and
+      subsequently incorporated within the Work.
+
+   2. Grant of Copyright License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      copyright license to reproduce, prepare Derivative Works of,
+      publicly display, publicly perform, sublicense, and distribute the
+      Work and such Derivative Works in Source or Object form.
+
+   3. Grant of Patent License. Subject to the terms and conditions of
+      this License, each Contributor hereby grants to You a perpetual,
+      worldwide, non-exclusive, no-charge, royalty-free, irrevocable
+      (except as stated in this section) patent license to make, have made,
+      use, offer to sell, sell, import, and otherwise transfer the Work,
+      where such license applies only to those patent claims licensable
+      by such Contributor that are necessarily infringed by their
+      Contribution(s) alone or by combination of their Contribution(s)
+      with the Work to which such Contribution(s) was submitted. If You
+      institute patent litigation against any entity (including a
+      cross-claim or counterclaim in a lawsuit) alleging that the Work
+      or a Contribution incorporated within the Work constitutes direct
+      or contributory patent infringement, then any patent licenses
+      granted to You under this License for that Work shall terminate
+      as of the date such litigation is filed.
+
+   4. Redistribution. You may reproduce and distribute copies of the
+      Work or Derivative Works thereof in any medium, with or without
+      modifications, and in Source or Object form, provided that You
+      meet the following conditions:
+
+      (a) You must give any other recipients of the Work or
+          Derivative Works a copy of this License; and
+
+      (b) You must cause any modified files to carry prominent notices
+          stating that You changed the files; and
+
+      (c) You must retain, in the Source form of any Derivative Works
+          that You distribute, all copyright, patent, trademark, and
+          attribution notices from the Source form of the Work,
+          excluding those notices that do not pertain to any part of
+          the Derivative Works; and
+
+      (d) If the Work includes a "NOTICE" text file as part of its
+          distribution, then any Derivative Works that You distribute must
+          include a readable copy of the attribution notices contained
+          within such NOTICE file, excluding those notices that do not
+          pertain to any part of the Derivative Works, in at least one
+          of the following places: within a NOTICE text file distributed
+          as part of the Derivative Works; within the Source form or
+          documentation, if provided along with the Derivative Works; or,
+          within a display generated by the Derivative Works, if and
+          wherever such third-party notices normally appear. The contents
+          of the NOTICE file are for informational purposes only and
+          do not modify the License. You may add Your own attribution
+          notices within Derivative Works that You distribute, alongside
+          or as an addendum to the NOTICE text from the Work, provided
+          that such additional attribution notices cannot be construed
+          as modifying the License.
+
+      You may add Your own copyright statement to Your modifications and
+      may provide additional or different license terms and conditions
+      for use, reproduction, or distribution of Your modifications, or
+      for any such Derivative Works as a whole, provided Your use,
+      reproduction, and distribution of the Work otherwise complies with
+      the conditions stated in this License.
+
+   5. Submission of Contributions. Unless You explicitly state otherwise,
+      any Contribution intentionally submitted for inclusion in the Work
+      by You to the Licensor shall be under the terms and conditions of
+      this License, without any additional terms or conditions.
+      Notwithstanding the above, nothing herein shall supersede or modify
+      the terms of any separate license agreement you may have executed
+      with Licensor regarding such Contributions.
+
+   6. Trademarks. This License does not grant permission to use the trade
+      names, trademarks, service marks, or product names of the Licensor,
+      except as required for reasonable and customary use in describing the
+      origin of the Work and reproducing the content of the NOTICE file.
+
+   7. Disclaimer of Warranty. Unless required by applicable law or
+      agreed to in writing, Licensor provides the Work (and each
+      Contributor provides its Contributions) on an "AS IS" BASIS,
+      WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or
+      implied, including, without limitation, any warranties or conditions
+      of TITLE, NON-INFRINGEMENT, MERCHANTABILITY, or FITNESS FOR A
+      PARTICULAR PURPOSE. You are solely responsible for determining the
+      appropriateness of using or redistributing the Work and assume any
+      risks associated with Your exercise of permissions under this License.
+
+   8. Limitation of Liability. In no event and under no legal theory,
+      whether in tort (including negligence), contract, or otherwise,
+      unless required by applicable law (such as deliberate and grossly
+      negligent acts) or agreed to in writing, shall any Contributor be
+      liable to You for damages, including any direct, indirect, special,
+      incidental, or consequential damages of any character arising as a
+      result of this License or out of the use or inability to use the
+      Work (including but not limited to damages for loss of goodwill,
+      work stoppage, computer failure or malfunction, or any and all
+      other commercial damages or losses), even if such Contributor
+      has been advised of the possibility of such damages.
+
+   9. Accepting Warranty or Additional Liability. While redistributing
+      the Work or Derivative Works thereof, You may choose to offer,
+      and charge a fee for, acceptance of support, warranty, indemnity,
+      or other liability obligations and/or rights consistent with this
+      License. However, in accepting such obligations, You may act only
+      on Your own behalf and on Your sole responsibility, not on behalf
+      of any other Contributor, and only if You agree to indemnify,
+      defend, and hold each Contributor harmless for any liability
+      incurred by, or claims asserted against, such Contributor by reason
+      of your accepting any such warranty or additional liability.
+
+   END OF TERMS AND CONDITIONS
+
+   APPENDIX: How to apply the Apache License to your work.
+
+      To apply the Apache License to your work, attach the following
+      boilerplate notice, with the fields enclosed by brackets "[]"
+      replaced with your own identifying information. (Don't include
+      the brackets!)  The text should be enclosed in the appropriate
+      comment syntax for the file format. We also recommend that a
+      file or class name and description of purpose be included on the
+      same "printed page" as the copyright notice for easier
+      identification within third-party archives.
+
+   Copyright [yyyy] [name of copyright owner]
+
+   Licensed under the Apache License, Version 2.0 (the "License");
+   you may not use this file except in compliance with the License.
+   You may obtain a copy of the License at
+
+       http://www.apache.org/licenses/LICENSE-2.0
+
+   Unless required by applicable law or agreed to in writing, software
+   distributed under the License is distributed on an "AS IS" BASIS,
+   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+   See the License for the specific language governing permissions and
+   limitations under the License.

Makefile

@@ -0,0 +1,20 @@
+.PHONY: quality style test
+
+check_dirs := tests src benchmarks utils
+
+# Check that source code meets quality standards
+
+quality:
+	ruff check $(check_dirs) setup.py  # linter
+	ruff format --check $(check_dirs) setup.py # formatter
+
+# Format source code automatically
+
+style:
+	ruff check --fix $(check_dirs) setup.py # linter
+	ruff format $(check_dirs) setup.py # formatter
+
+# Run tests for the library
+
+test:
+	python -m pytest -n auto --dist=loadfile -s -v ./tests/

README.md

@@ -0,0 +1,299 @@
+<p align="center">
+  <picture>
+    <source media="(prefers-color-scheme: dark)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/datasets-logo-dark.svg">
+    <source media="(prefers-color-scheme: light)" srcset="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/datasets-logo-light.svg">
+    <img alt="Hugging Face Datasets Library" src="https://huggingface.co/datasets/huggingface/documentation-images/raw/main/datasets-logo-light.svg" width="352" height="59" style="max-width: 100%;">
+  </picture>
+  <br/>
+  <br/>
+</p>
+
+<p align="center">
+    <a href="https://github.com/huggingface/datasets/actions/workflows/ci.yml?query=branch%3Amain"><img alt="Build" src="https://github.com/huggingface/datasets/actions/workflows/ci.yml/badge.svg?branch=main"></a>
+    <a href="https://github.com/huggingface/datasets/blob/main/LICENSE"><img alt="GitHub" src="https://img.shields.io/github/license/huggingface/datasets.svg?color=blue"></a>
+    <a href="https://huggingface.co/docs/datasets/index.html"><img alt="Documentation" src="https://img.shields.io/website/http/huggingface.co/docs/datasets/index.html.svg?down_color=red&down_message=offline&up_message=online"></a>
+    <a href="https://github.com/huggingface/datasets/releases"><img alt="GitHub release" src="https://img.shields.io/github/release/huggingface/datasets.svg"></a>
+    <a href="https://huggingface.co/datasets/"><img alt="Number of datasets" src="https://img.shields.io/endpoint?url=https://huggingface.co/api/shields/datasets&color=brightgreen"></a>
+    <a href="CODE_OF_CONDUCT.md"><img alt="Contributor Covenant" src="https://img.shields.io/badge/Contributor%20Covenant-2.0-4baaaa.svg"></a>
+    <a href="https://zenodo.org/badge/latestdoi/250213286"><img src="https://zenodo.org/badge/250213286.svg" alt="DOI"></a>
+</p>
+
+🤗 Datasets is a lightweight library providing **two** main features:
+
+- **one-line dataloaders for many public datasets**: one-liners to download and pre-process any of the ![number of datasets](https://img.shields.io/endpoint?url=https://huggingface.co/api/shields/datasets&color=brightgreen) major public datasets (image datasets, audio datasets, text datasets in 467 languages and dialects, 3D medical images, video datasets, agent traces, etc.) provided on the [HuggingFace Datasets Hub](https://huggingface.co/datasets). With a simple command like `squad_dataset = load_dataset("rajpurkar/squad")`, get any of these datasets ready to use in a dataloader for training/evaluating a ML model (Numpy/Pandas/PyTorch/TensorFlow/JAX/Polars),
+- **efficient data pre-processing**: simple, fast and reproducible data pre-processing for the public datasets as well as your own local datasets in CSV, JSON, JSONL, Parquet, HDF5, XML, text, PNG, JPEG, WAV, MP3, PDF, NIfTI, and more. With simple commands like `processed_dataset = dataset.map(process_example)`, efficiently prepare the dataset for inspection and ML model evaluation and training.
+
+[🎓 **Documentation**](https://huggingface.co/docs/datasets/) [🔎 **Find a dataset in the Hub**](https://huggingface.co/datasets) [🌟 **Share a dataset on the Hub**](https://huggingface.co/docs/datasets/share)
+
+<h3 align="center">
+    <a href="https://hf.co/course"><img src="https://raw.githubusercontent.com/huggingface/datasets/main/docs/source/imgs/course_banner.png"></a>
+</h3>
+
+# 🚀 Key Features
+
+🤗 Datasets is designed to let the community easily add and share new datasets, and provides powerful capabilities for data manipulation:
+
+| Feature | Description |
+|---------|-------------|
+| 📦 **One-line dataset loading** | Load AI-ready datasets from the [Hugging Face Hub](https://huggingface.co/datasets) or local files with `load_dataset()` |
+| 🔍 **Multiple formats** | Native support for CSV, JSON, JSONL, Parquet, Arrow, XML, Text, Webdataset, and more |
+| 🖼️ **Multi-modal data** | Built-in support for text, audio, image, video, PDF, and NIfTI (3D medical) data |
+| 🚀 **Streaming mode** | Stream datasets without downloading — iterate over data on-the-fly with `streaming=True` (now up to **100x faster** with Xet backend) |
+| 💾 **HF Storage Buckets** | Read and write directly from/to [Hugging Face Storage Buckets](https://huggingface.co/docs/hub/storage-buckets) for mutable, large-scale raw data |
+| 🧠 **AI Agent Traces** | Load and process AI agent traces (prompts, tool calls, responses) from the Hub |
+| ⚡ **Apache Arrow backend** | Zero-copy memory-mapped storage — datasets naturally free you from RAM limitations |
+| 🔄 **Smart caching** | Never wait for your data to process twice — cached results are automatically reused |
+| 📊 **Multi-framework interoperability** | Native conversion to/from NumPy, Pandas, Polars, Arrow, PyTorch, TensorFlow, JAX, and Spark |
+| 🏎️ **Multi-processing** | Fast parallel data processing with `map(num_proc=N)` |
+| 🔎 **Search & index** | Built-in FAISS and Elasticsearch index support for similarity search |
+| 📦 **JSON type** | Flexible JSON/structured data support with `Json()` feature type |
+
+# Installation
+
+## With pip
+
+🤗 Datasets can be installed from PyPi and should be installed in a virtual environment (venv or conda for instance):
+
+```bash
+pip install datasets
+```
+
+For the latest development version:
+
+```bash
+pip install "datasets @ git+https://github.com/huggingface/datasets.git"
+```
+
+## With conda
+
+```bash
+conda install -c huggingface -c conda-forge datasets
+```
+
+## Optional dependencies
+
+🤗 Datasets supports various optional features via extras:
+
+```bash
+# For audio (torchcodec)
+pip install datasets[audio]
+
+# For image/video (Pillow, torchcodec)
+pip install datasets[vision]
+
+# For PDFs/NIfTI (pdfplumber, nibabel)
+pip install datasets[pdfs,nibabel]
+
+# For PyTorch/TensorFlow/JAX integration
+pip install datasets[torch,tensorflow,jax]
+
+```
+
+For more details on installation, check the [installation page](https://huggingface.co/docs/datasets/installation).
+
+# Quick Start
+
+🤗 Datasets is made to be very simple to use — the API is centered around a single function, `datasets.load_dataset(dataset_name, **kwargs)`, that instantiates a dataset.
+
+Here is a quick example:
+
+```python
+from datasets import load_dataset
+
+# Load a dataset and print the first example in the training set
+squad_dataset = load_dataset('rajpurkar/squad')
+print(squad_dataset['train'][0])
+
+# Process the dataset - add a column with the length of the context texts
+dataset_with_length = squad_dataset.map(lambda x: {"length": len(x["context"])})
+
+# Tokenize the context texts (using a tokenizer from the 🤗 Transformers library)
+from transformers import AutoTokenizer
+tokenizer = AutoTokenizer.from_pretrained('bert-base-cased')
+
+tokenized_dataset = squad_dataset.map(lambda x: tokenizer(x['context']), batched=True)
+
+# Tokenize chat conversations with a chat template (using a model that supports chat templates)
+# This is useful for fine-tuning instruction/chat models
+
+# Load a popular chat dataset (ultrachat_200k contains ~200k AI assistant conversations)
+chat_dataset = load_dataset('HuggingFaceH4/ultrachat_200k', split='train_sft')
+
+chat_tokenizer = AutoTokenizer.from_pretrained('Qwen/Qwen2.5-7B-Instruct')
+
+def tokenize_chat(examples):
+    # Apply the chat template and tokenize in one step
+    return chat_tokenizer.apply_chat_template(examples["messages"])
+
+tokenized_chat_dataset = chat_dataset.map(tokenize_chat, batched=True)
+```
+
+## Streaming mode
+
+If your dataset is bigger than your disk or if you don't want to wait to download the data, you can use streaming:
+
+```python
+# Stream the dataset without downloading anything
+image_dataset = load_dataset('timm/imagenet-1k-wds', streaming=True)
+for example in image_dataset["train"]:
+    print(example["image"])
+    break
+```
+
+## Multi-modal data
+
+🤗 Datasets supports a wide variety of data types out of the box:
+
+```python
+# Audio dataset
+dataset = load_dataset("openslr/librispeech_asr", "clean")
+
+# Image dataset
+dataset = load_dataset("ILSVRC/imagenet-1k")
+
+# Video dataset
+dataset = load_dataset("Shofo/shofo-tiktok-general-small")
+
+# PDF documents
+dataset = load_dataset("pixparse/pdfa-eng-wds")
+
+# NIfTI (3D medical imaging)
+dataset = load_dataset("dartbrains/localizer", "betas")
+```
+
+## From local files
+
+```python
+# Load from local CSV
+dataset = load_dataset('csv', data_files='my_data.csv')
+
+# Load from local Parquet
+dataset = load_dataset('parquet', data_files='data/*.parquet')
+
+# Load from a local directory (auto-detect format)
+dataset = load_dataset('./path/to/data')
+```
+
+## From Python objects
+
+```python
+from datasets import Dataset
+
+# From a dictionary
+dataset = Dataset.from_dict({"text": ["Hello world", "How are you?"]})
+
+# From a list
+dataset = Dataset.from_list([{"text": "Hello world"}, {"text": "How are you?"}])
+
+# From Pandas
+import pandas as pd
+df = pd.DataFrame({"col1": [1, 2, 3], "col2": ["a", "b", "c"]})
+dataset = Dataset.from_pandas(df)
+
+# From a generator
+def gen():
+    for i in range(10):
+        yield {"value": i}
+dataset = Dataset.from_generator(gen)
+```
+
+For more details on using the library, check the [quick start guide](https://huggingface.co/docs/datasets/quickstart) and the specific pages on:
+
+- [Loading a dataset](https://huggingface.co/docs/datasets/loading)
+- [What's in a Dataset](https://huggingface.co/docs/datasets/access)
+- [Processing data with 🤗 Datasets](https://huggingface.co/docs/datasets/process)
+  - [Processing audio data](https://huggingface.co/docs/datasets/audio_process)
+  - [Processing image data](https://huggingface.co/docs/datasets/image_process)
+  - [Processing text data](https://huggingface.co/docs/datasets/nlp_process)
+  - [Processing PDF data](https://huggingface.co/docs/datasets/pdf_process)
+  - [Processing video data](https://huggingface.co/docs/datasets/video_process)
+- [Streaming a dataset](https://huggingface.co/docs/datasets/stream)
+
+# Core Classes
+
+The library provides two main dataset classes:
+
+| Class | Description |
+|-------|-------------|
+| `Dataset` | In-memory / memory-mapped dataset backed by Apache Arrow. Supports indexing, slicing, random access and caching. |
+| `IterableDataset` | Lazy, streamable dataset for large-scale / out-of-core processing. Supports streaming and infinite iteration. |
+
+Both are wrapped in `DatasetDict` / `IterableDatasetDict` for multi-split datasets (e.g., train/test/val).
+
+# Add a new dataset to the Hub
+
+We have a very detailed step-by-step guide to add a new dataset to the ![number of datasets](https://img.shields.io/endpoint?url=https://huggingface.co/api/shields/datasets&color=brightgreen) datasets already provided on the [HuggingFace Datasets Hub](https://huggingface.co/datasets).
+
+You can find:
+- [how to upload a dataset to the Hub using your web browser or Python](https://huggingface.co/docs/datasets/upload_dataset) and also
+- [how to upload it using Git](https://huggingface.co/docs/datasets/share).
+
+# Disclaimers
+
+You can use 🤗 Datasets to load datasets based on versioned git repositories maintained by the dataset authors. For reproducibility reasons, we ask users to pin the `revision` of the repositories they use.
+
+If you're a dataset owner and wish to update any part of it (description, citation, license, etc.), or do not want your dataset to be included in the Hugging Face Hub, please get in touch by opening a discussion or a pull request in the Community tab of the dataset page. Thanks for your contribution to the ML community!
+
+# Contributing
+
+We welcome contributions! Please see our [Contributing Guide](CONTRIBUTING.md) for details on:
+
+- How to submit issues and pull requests
+- Code style guidelines (we use [Ruff](https://docs.astral.sh/ruff/))
+- Testing requirements
+- Documentation standards
+
+# BibTeX
+
+If you want to cite our 🤗 Datasets library, you can use our [paper](https://huggingface.co/papers/2109.02846):
+
+```bibtex
+@inproceedings{lhoest-etal-2021-datasets,
+    title = "Datasets: A Community Library for Natural Language Processing",
+    author = "Lhoest, Quentin  and
+      Villanova del Moral, Albert  and
+      Jernite, Yacine  and
+      Thakur, Abhishek  and
+      von Platen, Patrick  and
+      Patil, Suraj  and
+      Chaumond, Julien  and
+      Drame, Mariama  and
+      Plu, Julien  and
+      Tunstall, Lewis  and
+      Davison, Joe  and
+      {\v{S}}a{\v{s}}ko, Mario  and
+      Chhablani, Gunjan  and
+      Malik, Bhavitvya  and
+      Brandeis, Simon  and
+      Le Scao, Teven  and
+      Sanh, Victor  and
+      Xu, Canwen  and
+      Patry, Nicolas  and
+      McMillan-Major, Angelina  and
+      Schmid, Philipp  and
+      Gugger, Sylvain  and
+      Delangue, Cl{\'e}ment  and
+      Matussi{\`e}re, Th{\'e}o  and
+      Debut, Lysandre  and
+      Bekman, Stas  and
+      Cistac, Pierric  and
+      Goehringer, Thibault  and
+      Mustar, Victor  and
+      Lagunas, Fran{\c{c}}ois  and
+      Rush, Alexander  and
+      Wolf, Thomas",
+    booktitle = "Proceedings of the 2021 Conference on Empirical Methods in Natural Language Processing: System Demonstrations",
+    month = nov,
+    year = "2021",
+    address = "Online and Punta Cana, Dominican Republic",
+    publisher = "Association for Computational Linguistics",
+    url = "https://aclanthology.org/2021.emnlp-demo.21",
+    pages = "175--184",
+    abstract = "The scale, variety, and quantity of publicly-available NLP datasets has grown rapidly as researchers propose new tasks, larger models, and novel benchmarks. Datasets is a community library for contemporary NLP designed to support this ecosystem. Datasets aims to standardize end-user interfaces, versioning, and documentation, while providing a lightweight front-end that behaves similarly for small datasets as for internet-scale corpora. The design of the library incorporates a distributed, community-driven approach to adding datasets and documenting usage. After a year of development, the library now includes more than 650 unique datasets, has more than 250 contributors, and has helped support a variety of novel cross-dataset research projects and shared tasks. The library is available at https://github.com/huggingface/datasets.",
+    eprint={2109.02846},
+    archivePrefix={arXiv},
+    primaryClass={cs.CL},
+}
+```
+
+If you need to cite a specific version of our 🤗 Datasets library for reproducibility, you can use the corresponding version Zenodo DOI from this [list](https://zenodo.org/search?q=conceptrecid:%224817768%22&sort=-version&all_versions=True).

README.wehub.md

@@ -0,0 +1,7 @@
+# WeHub 来源说明
+
+- 原始项目：`huggingface/datasets`
+- 原始仓库：https://github.com/huggingface/datasets
+- 导入方式：上游默认分支的最新快照
+- 原作者、版权和许可证信息以原始仓库及本仓库 LICENSE 为准
+- 本文件仅用于记录来源，不代表 WeHub 是原项目作者

SECURITY.md

@@ -0,0 +1,33 @@
+# Security Policy
+
+## Supported Versions
+<!--
+Use this section to tell people about which versions of your project are
+currently being supported with security updates.
+
+| Version | Supported          |
+| ------- | ------------------ |
+| 5.1.x   | :white_check_mark: |
+| 5.0.x   | :x:                |
+| 4.0.x   | :white_check_mark: |
+| < 4.0   | :x:                |
+-->
+
+Each major version is currently being supported with security updates.
+
+| Version | Supported          |
+|---------|--------------------|
+| 1.x.x   | :white_check_mark: |
+| 2.x.x   | :white_check_mark: |
+
+
+## Reporting a Vulnerability
+<!--
+Use this section to tell people how to report a vulnerability.
+
+Tell them where to go, how often they can expect to get an update on a
+reported vulnerability, what to expect if the vulnerability is accepted or
+declined, etc.
+-->
+
+To report a security vulnerability, please contact: security@huggingface.co

benchmarks/benchmark_array_xd.py

@@ -0,0 +1,142 @@
+import json
+import os
+import tempfile
+
+import datasets
+from datasets.arrow_writer import ArrowWriter
+from datasets.features import Array2D
+from utils import generate_examples, get_duration
+
+
+SHAPE_TEST_1 = (30, 487)
+SHAPE_TEST_2 = (36, 1024)
+SPEED_TEST_SHAPE = (100, 100)
+SPEED_TEST_N_EXAMPLES = 100
+
+DEFAULT_FEATURES = datasets.Features(
+    {"text": Array2D(SHAPE_TEST_1, dtype="float32"), "image": Array2D(SHAPE_TEST_2, dtype="float32")}
+)
+
+RESULTS_BASEPATH, RESULTS_FILENAME = os.path.split(__file__)
+RESULTS_FILE_PATH = os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
+
+
+@get_duration
+def write(my_features, dummy_data, tmp_dir):
+    with ArrowWriter(features=my_features, path=os.path.join(tmp_dir, "beta.arrow")) as writer:
+        for key, record in dummy_data:
+            example = my_features.encode_example(record)
+            writer.write(example)
+        num_examples, num_bytes = writer.finalize()
+
+
+@get_duration
+def read_unformated(feats, tmp_dir):
+    dataset = datasets.Dataset.from_file(
+        filename=os.path.join(tmp_dir, "beta.arrow"), info=datasets.DatasetInfo(features=feats)
+    )
+    for _ in dataset:
+        pass
+
+
+@get_duration
+def read_formatted_as_numpy(feats, tmp_dir):
+    dataset = datasets.Dataset.from_file(
+        filename=os.path.join(tmp_dir, "beta.arrow"), info=datasets.DatasetInfo(features=feats)
+    )
+    dataset.set_format("numpy")
+    for _ in dataset:
+        pass
+
+
+@get_duration
+def read_batch_unformated(feats, tmp_dir):
+    batch_size = 10
+    dataset = datasets.Dataset.from_file(
+        filename=os.path.join(tmp_dir, "beta.arrow"), info=datasets.DatasetInfo(features=feats)
+    )
+    for i in range(0, len(dataset), batch_size):
+        _ = dataset[i : i + batch_size]
+
+
+@get_duration
+def read_batch_formatted_as_numpy(feats, tmp_dir):
+    batch_size = 10
+    dataset = datasets.Dataset.from_file(
+        filename=os.path.join(tmp_dir, "beta.arrow"), info=datasets.DatasetInfo(features=feats)
+    )
+    dataset.set_format("numpy")
+    for i in range(0, len(dataset), batch_size):
+        _ = dataset[i : i + batch_size]
+
+
+@get_duration
+def read_col_unformated(feats, tmp_dir):
+    dataset = datasets.Dataset.from_file(
+        filename=os.path.join(tmp_dir, "beta.arrow"), info=datasets.DatasetInfo(features=feats)
+    )
+    for col in feats:
+        _ = dataset[col]
+
+
+@get_duration
+def read_col_formatted_as_numpy(feats, tmp_dir):
+    dataset = datasets.Dataset.from_file(
+        filename=os.path.join(tmp_dir, "beta.arrow"), info=datasets.DatasetInfo(features=feats)
+    )
+    dataset.set_format("numpy")
+    for col in feats:
+        _ = dataset[col]
+
+
+def benchmark_array_xd():
+    times = {}
+    read_functions = (
+        read_unformated,
+        read_formatted_as_numpy,
+        read_batch_unformated,
+        read_batch_formatted_as_numpy,
+        read_col_unformated,
+        read_col_formatted_as_numpy,
+    )
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        feats = datasets.Features({"image": Array2D(SPEED_TEST_SHAPE, dtype="float32")})
+        data = generate_examples(features=feats, num_examples=SPEED_TEST_N_EXAMPLES)
+        times["write_array2d"] = write(feats, data, tmp_dir)
+        for read_func in read_functions:
+            times[read_func.__name__ + " after write_array2d"] = read_func(feats, tmp_dir)
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        # don't use fixed length for fair comparison
+        # feats = datasets.Features(
+        #     {"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[1]), SPEED_TEST_SHAPE[0])}
+        # )
+        feats = datasets.Features({"image": datasets.Sequence(datasets.Sequence(datasets.Value("float32")))})
+        data = generate_examples(
+            features=feats, num_examples=SPEED_TEST_N_EXAMPLES, seq_shapes={"image": SPEED_TEST_SHAPE}
+        )
+        times["write_nested_sequence"] = write(feats, data, tmp_dir)
+        for read_func in read_functions:
+            times[read_func.__name__ + " after write_nested_sequence"] = read_func(feats, tmp_dir)
+
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        # don't use fixed length for fair comparison
+        # feats = datasets.Features(
+        #     {"image": datasets.Sequence(datasets.Value("float32"), SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1])}
+        # )
+        feats = datasets.Features({"image": datasets.Sequence(datasets.Value("float32"))})
+        data = generate_examples(
+            features=feats,
+            num_examples=SPEED_TEST_N_EXAMPLES,
+            seq_shapes={"image": [SPEED_TEST_SHAPE[0] * SPEED_TEST_SHAPE[1]]},
+        )
+        times["write_flattened_sequence"] = write(feats, data, tmp_dir)
+        for read_func in read_functions:
+            times[read_func.__name__ + " after write_flattened_sequence"] = read_func(feats, tmp_dir)
+
+    with open(RESULTS_FILE_PATH, "wb") as f:
+        f.write(json.dumps(times).encode("utf-8"))
+
+
+if __name__ == "__main__":  # useful to run the profiler
+    benchmark_array_xd()

benchmarks/benchmark_getitem_100B.py

@@ -0,0 +1,78 @@
+import json
+import os
+from dataclasses import dataclass
+
+import numpy as np
+import pyarrow as pa
+
+import datasets
+from utils import get_duration
+
+
+SPEED_TEST_N_EXAMPLES = 100_000_000_000
+SPEED_TEST_CHUNK_SIZE = 10_000
+
+RESULTS_BASEPATH, RESULTS_FILENAME = os.path.split(__file__)
+RESULTS_FILE_PATH = os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
+
+
+def generate_100B_dataset(num_examples: int, chunk_size: int) -> datasets.Dataset:
+    table = pa.Table.from_pydict({"col": [0] * chunk_size})
+    table = pa.concat_tables([table] * (num_examples // chunk_size))
+    return datasets.Dataset(table, fingerprint="table_100B")
+
+
+@dataclass
+class RandIter:
+    low: int
+    high: int
+    size: int
+    seed: int
+
+    def __post_init__(self):
+        rng = np.random.default_rng(self.seed)
+        self._sampled_values = rng.integers(low=self.low, high=self.high, size=self.size).tolist()
+
+    def __iter__(self):
+        return iter(self._sampled_values)
+
+    def __len__(self):
+        return self.size
+
+
+@get_duration
+def get_first_row(dataset: datasets.Dataset):
+    _ = dataset[0]
+
+
+@get_duration
+def get_last_row(dataset: datasets.Dataset):
+    _ = dataset[-1]
+
+
+@get_duration
+def get_batch_of_1024_rows(dataset: datasets.Dataset):
+    _ = dataset[range(len(dataset) // 2, len(dataset) // 2 + 1024)]
+
+
+@get_duration
+def get_batch_of_1024_random_rows(dataset: datasets.Dataset):
+    _ = dataset[RandIter(0, len(dataset), 1024, seed=42)]
+
+
+def benchmark_table_100B():
+    times = {"num examples": SPEED_TEST_N_EXAMPLES}
+    functions = (get_first_row, get_last_row, get_batch_of_1024_rows, get_batch_of_1024_random_rows)
+    print("generating dataset")
+    dataset = generate_100B_dataset(num_examples=SPEED_TEST_N_EXAMPLES, chunk_size=SPEED_TEST_CHUNK_SIZE)
+    print("Functions")
+    for func in functions:
+        print(func.__name__)
+        times[func.__name__] = func(dataset)
+
+    with open(RESULTS_FILE_PATH, "wb") as f:
+        f.write(json.dumps(times).encode("utf-8"))
+
+
+if __name__ == "__main__":  # useful to run the profiler
+    benchmark_table_100B()

benchmarks/benchmark_indices_mapping.py

@@ -0,0 +1,60 @@
+import json
+import os
+import tempfile
+
+import datasets
+from utils import generate_example_dataset, get_duration
+
+
+SPEED_TEST_N_EXAMPLES = 500_000
+
+RESULTS_BASEPATH, RESULTS_FILENAME = os.path.split(__file__)
+RESULTS_FILE_PATH = os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
+
+
+@get_duration
+def select(dataset: datasets.Dataset):
+    _ = dataset.select(range(0, len(dataset), 2))
+
+
+@get_duration
+def sort(dataset: datasets.Dataset):
+    _ = dataset.sort("numbers")
+
+
+@get_duration
+def shuffle(dataset: datasets.Dataset):
+    _ = dataset.shuffle()
+
+
+@get_duration
+def train_test_split(dataset: datasets.Dataset):
+    _ = dataset.train_test_split(0.1)
+
+
+@get_duration
+def shard(dataset: datasets.Dataset, num_shards=10):
+    for shard_id in range(num_shards):
+        _ = dataset.shard(num_shards, shard_id)
+
+
+def benchmark_indices_mapping():
+    times = {"num examples": SPEED_TEST_N_EXAMPLES}
+    functions = (select, sort, shuffle, train_test_split, shard)
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        print("generating dataset")
+        features = datasets.Features({"text": datasets.Value("string"), "numbers": datasets.Value("float32")})
+        dataset = generate_example_dataset(
+            os.path.join(tmp_dir, "dataset.arrow"), features, num_examples=SPEED_TEST_N_EXAMPLES
+        )
+        print("Functions")
+        for func in functions:
+            print(func.__name__)
+            times[func.__name__] = func(dataset)
+
+    with open(RESULTS_FILE_PATH, "wb") as f:
+        f.write(json.dumps(times).encode("utf-8"))
+
+
+if __name__ == "__main__":  # useful to run the profiler
+    benchmark_indices_mapping()

benchmarks/benchmark_iterating.py

@@ -0,0 +1,98 @@
+import json
+import os
+import tempfile
+
+import datasets
+from utils import generate_example_dataset, get_duration
+
+
+SPEED_TEST_N_EXAMPLES = 50_000
+SMALL_TEST = 5_000
+
+RESULTS_BASEPATH, RESULTS_FILENAME = os.path.split(__file__)
+RESULTS_FILE_PATH = os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
+
+
+@get_duration
+def read(dataset: datasets.Dataset, length):
+    for i in range(length):
+        _ = dataset[i]
+
+
+@get_duration
+def read_batch(dataset: datasets.Dataset, length, batch_size):
+    for i in range(0, len(dataset), batch_size):
+        _ = dataset[i : i + batch_size]
+
+
+@get_duration
+def read_formatted(dataset: datasets.Dataset, length, type):
+    with dataset.formatted_as(type=type):
+        for i in range(length):
+            _ = dataset[i]
+
+
+@get_duration
+def read_formatted_batch(dataset: datasets.Dataset, length, batch_size, type):
+    with dataset.formatted_as(type=type):
+        for i in range(0, length, batch_size):
+            _ = dataset[i : i + batch_size]
+
+
+def benchmark_iterating():
+    times = {"num examples": SPEED_TEST_N_EXAMPLES}
+    functions = [
+        (read, {"length": SMALL_TEST}),
+        (read, {"length": SPEED_TEST_N_EXAMPLES}),
+        (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 10}),
+        (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 100}),
+        (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 1_000}),
+        (read_formatted, {"type": "numpy", "length": SMALL_TEST}),
+        (read_formatted, {"type": "pandas", "length": SMALL_TEST}),
+        (read_formatted, {"type": "torch", "length": SMALL_TEST}),
+        (read_formatted, {"type": "tensorflow", "length": SMALL_TEST}),
+        (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 10}),
+        (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 1_000}),
+    ]
+
+    functions_shuffled = [
+        (read, {"length": SMALL_TEST}),
+        (read, {"length": SPEED_TEST_N_EXAMPLES}),
+        (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 10}),
+        (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 100}),
+        (read_batch, {"length": SPEED_TEST_N_EXAMPLES, "batch_size": 1_000}),
+        (read_formatted, {"type": "numpy", "length": SMALL_TEST}),
+        (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 10}),
+        (read_formatted_batch, {"type": "numpy", "length": SMALL_TEST, "batch_size": 1_000}),
+    ]
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        print("generating dataset")
+        features = datasets.Features(
+            {"list": datasets.Sequence(datasets.Value("float32")), "numbers": datasets.Value("float32")}
+        )
+        dataset = generate_example_dataset(
+            os.path.join(tmp_dir, "dataset.arrow"),
+            features,
+            num_examples=SPEED_TEST_N_EXAMPLES,
+            seq_shapes={"list": (100,)},
+        )
+        print("first set of iterations")
+        for func, kwargs in functions:
+            print(func.__name__, str(kwargs))
+            times[func.__name__ + " " + " ".join(str(v) for v in kwargs.values())] = func(dataset, **kwargs)
+
+        print("shuffling dataset")
+        dataset = dataset.shuffle()
+        print("Second set of iterations (after shuffling")
+        for func, kwargs in functions_shuffled:
+            print("shuffled ", func.__name__, str(kwargs))
+            times["shuffled " + func.__name__ + " " + " ".join(str(v) for v in kwargs.values())] = func(
+                dataset, **kwargs
+            )
+
+    with open(RESULTS_FILE_PATH, "wb") as f:
+        f.write(json.dumps(times).encode("utf-8"))
+
+
+if __name__ == "__main__":  # useful to run the profiler
+    benchmark_iterating()

benchmarks/benchmark_map_filter.py

@@ -0,0 +1,71 @@
+import json
+import os
+import tempfile
+
+import transformers
+
+import datasets
+from utils import generate_example_dataset, get_duration
+
+
+SPEED_TEST_N_EXAMPLES = 500_000
+
+RESULTS_BASEPATH, RESULTS_FILENAME = os.path.split(__file__)
+RESULTS_FILE_PATH = os.path.join(RESULTS_BASEPATH, "results", RESULTS_FILENAME.replace(".py", ".json"))
+
+
+@get_duration
+def map(dataset: datasets.Dataset, **kwargs):
+    _ = dataset.map(**kwargs)
+
+
+@get_duration
+def filter(dataset: datasets.Dataset, **kwargs):
+    _ = dataset.filter(**kwargs)
+
+
+def benchmark_map_filter():
+    times = {"num examples": SPEED_TEST_N_EXAMPLES}
+    with tempfile.TemporaryDirectory() as tmp_dir:
+        features = datasets.Features({"text": datasets.Value("string"), "numbers": datasets.Value("float32")})
+        dataset = generate_example_dataset(
+            os.path.join(tmp_dir, "dataset.arrow"), features, num_examples=SPEED_TEST_N_EXAMPLES
+        )
+
+        tokenizer = transformers.AutoTokenizer.from_pretrained("bert-base-cased", use_fast=True)
+
+        def tokenize(examples):
+            return tokenizer(examples["text"])
+
+        times["map identity"] = map(dataset)
+
+        times["map identity batched"] = map(dataset, batched=True)
+
+        times["map no-op batched"] = map(dataset, function=lambda x: None, batched=True)
+
+        with dataset.formatted_as(type="numpy"):
+            times["map no-op batched numpy"] = map(dataset, function=lambda x: None, batched=True)
+
+        with dataset.formatted_as(type="pandas"):
+            times["map no-op batched pandas"] = map(dataset, function=lambda x: None, batched=True)
+
+        with dataset.formatted_as(type="torch", columns="numbers"):
+            times["map no-op batched pytorch"] = map(dataset, function=lambda x: None, batched=True)
+
+        with dataset.formatted_as(type="tensorflow", columns="numbers"):
+            times["map no-op batched tensorflow"] = map(dataset, function=lambda x: None, batched=True)
+
+        times["map fast-tokenizer batched"] = map(dataset, function=tokenize, batched=True)
+
+        times["filter"] = filter(dataset)
+
+        # Activate later when tokenizer support batched inputs
+        # with dataset.formatted_as(type='numpy'):
+        #     times[func.__name__ + " fast-tokenizer batched numpy"] = func(dataset, function=tokenize, batched=True)
+
+    with open(RESULTS_FILE_PATH, "wb") as f:
+        f.write(json.dumps(times).encode("utf-8"))
+
+
+if __name__ == "__main__":  # useful to run the profiler
+    benchmark_map_filter()

benchmarks/format.py

@@ -0,0 +1,49 @@
+import json
+import sys
+
+
+def format_json_to_md(input_json_file, output_md_file):
+    with open(input_json_file, encoding="utf-8") as f:
+        results = json.load(f)
+
+    output_md = ["<details>", "<summary>Show updated benchmarks!</summary>", " "]
+
+    for benchmark_name in sorted(results):
+        benchmark_res = results[benchmark_name]
+
+        benchmark_file_name = benchmark_name.split("/")[-1]
+        output_md.append(f"### Benchmark: {benchmark_file_name}")
+
+        title = "| metric |"
+        lines = "|--------|"
+        value = "| new / old (diff) |"
+        for metric_name in sorted(benchmark_res):
+            metric_vals = benchmark_res[metric_name]
+            new_val = metric_vals["new"]
+            old_val = metric_vals.get("old", None)
+            dif_val = metric_vals.get("diff", None)
+
+            val_str = f" {new_val:f}" if isinstance(new_val, (int, float)) else "None"
+
+            if old_val is not None:
+                val_str += f" / {old_val:f}" if isinstance(old_val, (int, float)) else "None"
+            if dif_val is not None:
+                val_str += f" ({dif_val:f})" if isinstance(dif_val, (int, float)) else "None"
+
+            title += " " + metric_name + " |"
+            lines += "---|"
+            value += val_str + " |"
+
+        output_md += [title, lines, value, " "]
+
+    output_md.append("</details>")
+
+    with open(output_md_file, "w", encoding="utf-8") as f:
+        f.writelines("\n".join(output_md))
+
+
+if __name__ == "__main__":
+    input_json_file = sys.argv[1]
+    output_md_file = sys.argv[2]
+
+    format_json_to_md(input_json_file, output_md_file)

benchmarks/results/benchmark_array_xd.json

@@ -0,0 +1 @@
+{"write_array2d": 0.14168284999323077, "read_unformated after write_array2d": 0.04353281999647152, "read_formatted_as_numpy after write_array2d": 0.1285462469968479, "read_batch_unformated after write_array2d": 0.023109222995117307, "read_batch_formatted_as_numpy after write_array2d": 0.011352884990628809, "read_col_unformated after write_array2d": 0.037052362007671036, "read_col_formatted_as_numpy after write_array2d": 0.007985618998645805, "write_nested_sequence": 1.4927163410029607, "read_unformated after write_nested_sequence": 0.28319963401008863, "read_formatted_as_numpy after write_nested_sequence": 0.419271487990045, "read_batch_unformated after write_nested_sequence": 0.3234798710036557, "read_batch_formatted_as_numpy after write_nested_sequence": 0.03850809299910907, "read_col_unformated after write_nested_sequence": 0.29384092400141526, "read_col_formatted_as_numpy after write_nested_sequence": 0.004250421989127062, "write_flattened_sequence": 1.4521546780015342, "read_unformated after write_flattened_sequence": 0.25513897799828555, "read_formatted_as_numpy after write_flattened_sequence": 0.07564631900459062, "read_batch_unformated after write_flattened_sequence": 0.2758980469952803, "read_batch_formatted_as_numpy after write_flattened_sequence": 0.011008214991306886, "read_col_unformated after write_flattened_sequence": 0.25848906899045687, "read_col_formatted_as_numpy after write_flattened_sequence": 0.004328447001171298}

benchmarks/results/benchmark_getitem_100B.json

@@ -0,0 +1 @@
+{"num examples": 100000000000, "get_first_row": 0.00019991099999927542, "get_last_row": 5.4411000000698095e-05, "get_batch_of_1024_rows": 0.0004897069999998394, "get_batch_of_1024_random_rows": 0.01800621099999944}

benchmarks/results/benchmark_indices_mapping.json

@@ -0,0 +1 @@
+{"num examples": 500000, "select": 0.03741131999413483, "sort": 0.7371353159978753, "shuffle": 0.17655655200360343, "train_test_split": 0.29633847798686475, "shard": 0.01452581599005498}

benchmarks/results/benchmark_iterating.json

@@ -0,0 +1 @@
+{"num examples": 50000, "read 5000": 0.2152090710005723, "read 50000": 2.077654693988734, "read_batch 50000 10": 1.5041199039987987, "read_batch 50000 100": 1.5411947140091797, "read_batch 50000 1000": 1.4684901159926085, "read_formatted numpy 5000": 4.584776938994764, "read_formatted pandas 5000": 3.7457121399929747, "read_formatted torch 5000": 4.565676491998602, "read_formatted tensorflow 5000": 5.269861594992108, "read_formatted_batch numpy 5000 10": 0.4242750950070331, "read_formatted_batch numpy 5000 1000": 0.007607111998368055, "shuffled read 5000": 0.22604441999283154, "shuffled read 50000": 2.268928524994408, "shuffled read_batch 50000 10": 55.44462437101174, "shuffled read_batch 50000 100": 6.876476717996411, "shuffled read_batch 50000 1000": 2.1420724369963864, "shuffled read_formatted numpy 5000": 4.8052272600034485, "shuffled read_formatted_batch numpy 5000 10": 6.500664097999106, "shuffled read_formatted_batch numpy 5000 1000": 0.0754691059992183}

benchmarks/results/benchmark_map_filter.json

@@ -0,0 +1 @@
+{"num examples": 500000, "map identity": 10.19139202599763, "map identity batched": 0.6804238399927272, "map no-op batched": 0.5342009569867514, "map no-op batched numpy": 0.5792830920108827, "map no-op batched pandas": 0.4343639040016569, "map no-op batched pytorch": 0.5403374370071106, "map no-op batched tensorflow": 1.3869360350072384, "map fast-tokenizer batched": 8.074308118986664, "filter": 1.841787679004483}

benchmarks/utils.py

@@ -0,0 +1,64 @@
+import timeit
+
+import numpy as np
+
+import datasets
+from datasets.arrow_writer import ArrowWriter
+from datasets.features.features import _ArrayXD
+
+
+def get_duration(func):
+    def wrapper(*args, **kwargs):
+        starttime = timeit.default_timer()
+        _ = func(*args, **kwargs)
+        delta = timeit.default_timer() - starttime
+        return delta
+
+    wrapper.__name__ = func.__name__
+
+    return wrapper
+
+
+def generate_examples(features: dict, num_examples=100, seq_shapes=None):
+    dummy_data = []
+    seq_shapes = seq_shapes or {}
+    for i in range(num_examples):
+        example = {}
+        for col_id, (k, v) in enumerate(features.items()):
+            if isinstance(v, _ArrayXD):
+                data = np.random.rand(*v.shape).astype(v.dtype)
+            elif isinstance(v, datasets.Value):
+                if v.dtype == "string":
+                    data = "The small grey turtle was surprisingly fast when challenged."
+                else:
+                    data = np.random.randint(10, size=1).astype(v.dtype).item()
+            elif isinstance(v, datasets.Sequence):
+                while isinstance(v, datasets.Sequence):
+                    v = v.feature
+                shape = seq_shapes[k]
+                data = np.random.rand(*shape).astype(v.dtype)
+            example[k] = data
+
+        dummy_data.append((i, example))
+
+    return dummy_data
+
+
+def generate_example_dataset(dataset_path, features, num_examples=100, seq_shapes=None):
+    dummy_data = generate_examples(features, num_examples=num_examples, seq_shapes=seq_shapes)
+
+    with ArrowWriter(features=features, path=dataset_path) as writer:
+        for key, record in dummy_data:
+            example = features.encode_example(record)
+            writer.write(example)
+
+        num_final_examples, num_bytes = writer.finalize()
+
+    if not num_final_examples == num_examples:
+        raise ValueError(
+            f"Error writing the dataset, wrote {num_final_examples} examples but should have written {num_examples}."
+        )
+
+    dataset = datasets.Dataset.from_file(filename=dataset_path, info=datasets.DatasetInfo(features=features))
+
+    return dataset

docs/README.md

@@ -0,0 +1,260 @@
+<!---
+Copyright 2020 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+-->
+
+# Generating the documentation
+
+To generate the documentation, you first have to build it. Several packages are necessary to build the doc,
+you can install them with the following command, at the root of the code repository:
+
+```bash
+pip install -e ".[docs]"
+```
+
+Then you need to install our special tool that builds the documentation:
+
+```bash
+pip install git+https://github.com/huggingface/doc-builder
+```
+
+---
+**NOTE**
+
+You only need to generate the documentation to inspect it locally (if you're planning changes and want to
+check how they look before committing for instance). You don't have to `git commit` the built documentation.
+
+---
+
+## Building the documentation
+
+Once you have setup the `doc-builder` and additional packages, you can generate the documentation by typing
+the following command:
+
+```bash
+doc-builder build datasets docs/source/ --build_dir ~/tmp/test-build
+```
+
+You can adapt the `--build_dir` to set any temporary folder that you prefer. This command will create it and generate
+the MDX files that will be rendered as the documentation on the main website. You can inspect them in your favorite
+Markdown editor.
+
+## Previewing the documentation
+
+To preview the docs, first install the `watchdog` module with:
+
+```bash
+pip install watchdog
+```
+
+Then run the following command:
+
+```bash
+doc-builder preview datasets docs/source/
+```
+
+The docs will be viewable at [http://localhost:3000](http://localhost:3000). You can also preview the docs once you have opened a PR. You will see a bot add a comment to a link where the documentation with your changes lives.
+
+---
+**NOTE**
+
+The `preview` command only works with existing doc files. When you add a completely new file, you need to update `_toctree.yml` & restart `preview` command (`ctrl-c` to stop it & call `doc-builder preview ...` again).
+
+## Adding a new element to the navigation bar
+
+Accepted files are Markdown (.md or .mdx).
+
+Create a file with its extension and put it in the source directory. You can then link it to the toc-tree by putting
+the filename without the extension in the [`_toctree.yml`](https://github.com/huggingface/datasets/blob/main/docs/source/_toctree.yml) file.
+
+## Renaming section headers and moving sections
+
+It helps to keep the old links working when renaming the section header and/or moving sections from one document to another. This is because the old links are likely to be used in Issues, Forums and Social media and it'd make for a much more superior user experience if users reading those months later could still easily navigate to the originally intended information.
+
+Therefore we simply keep a little map of moved sections at the end of the document where the original section was. The key is to preserve the original anchor.
+
+So if you renamed a section from: "Section A" to "Section B", then you can add at the end of the file:
+
+```
+Sections that were moved:
+
+[ <a href="#section-b">Section A</a><a id="section-a"></a> ]
+```
+and of course if you moved it to another file, then:
+
+```
+Sections that were moved:
+
+[ <a href="../new-file#section-b">Section A</a><a id="section-a"></a> ]
+```
+
+Use the relative style to link to the new file so that the versioned docs continue to work.
+
+For an example of a rich moved sections set please see the very end of [the transformers Trainer doc](https://github.com/huggingface/transformers/blob/main/docs/source/en/main_classes/trainer.md).
+
+
+## Writing Documentation - Specification
+
+The `huggingface/datasets` documentation follows the
+[Google documentation](https://sphinxcontrib-napoleon.readthedocs.io/en/latest/example_google.html) style for docstrings,
+although we can write them directly in Markdown.
+
+### Adding a new tutorial
+
+Adding a new tutorial or section is done in two steps:
+
+- Add a new file under `./source`. This file can either be ReStructuredText (.rst) or Markdown (.md).
+- Link that file in `./source/_toctree.yml` on the correct toc-tree.
+
+Make sure to put your new file under the proper section. If you have a doubt, feel free to ask in a Github Issue or PR.
+
+### Writing source documentation
+
+Values that should be put in `code` should either be surrounded by backticks: \`like so\`. Note that argument names
+and objects like True, None or any strings should usually be put in `code`.
+
+When mentioning a class, function or method, it is recommended to use our syntax for internal links so that our tool
+adds a link to its documentation with this syntax: \[\`XXXClass\`\] or \[\`function\`\]. This requires the class or 
+function to be in the main package.
+
+If you want to create a link to some internal class or function, you need to
+provide its path. For instance: \[\`table.InMemoryTable\`\]. This will be converted into a link with
+`table.InMemoryTable` in the description. To get rid of the path and only keep the name of the object you are
+linking to in the description, add a ~: \[\`~table.InMemoryTable\`\] will generate a link with `InMemoryTable` in the description.
+
+The same works for methods so you can either use \[\`XXXClass.method\`\] or \[~\`XXXClass.method\`\].
+
+#### Defining arguments in a method
+
+Arguments should be defined with the `Args:` (or `Arguments:` or `Parameters:`) prefix, followed by a line return and
+an indentation. The argument should be followed by its type, with its shape if it is a tensor, a colon and its
+description:
+
+```
+    Args:
+        n_layers (`int`): The number of layers of the model.
+```
+
+If the description is too long to fit in one line, another indentation is necessary before writing the description
+after the argument.
+
+Here's an example showcasing everything so far:
+
+```
+    Args:
+        input_ids (`torch.LongTensor` of shape `(batch_size, sequence_length)`):
+            Indices of input sequence tokens in the vocabulary.
+
+            Indices can be obtained using [`AlbertTokenizer`]. See [`~PreTrainedTokenizer.encode`] and
+            [`~PreTrainedTokenizer.__call__`] for details.
+
+            [What are input IDs?](../glossary#input-ids)
+```
+
+For optional arguments or arguments with defaults we follow the following syntax: imagine we have a function with the
+following signature:
+
+```
+def my_function(x: str = None, a: float = 1):
+```
+
+then its documentation should look like this:
+
+```
+    Args:
+        x (`str`, *optional*):
+            This argument controls ...
+        a (`float`, *optional*, defaults to 1):
+            This argument is used to ...
+```
+
+Note that we always omit the "defaults to \`None\`" when None is the default for any argument. Also note that even
+if the first line describing your argument type and its default gets long, you can't break it into several lines. You can
+however write as many lines as you want in the indented description (see the example above with `input_ids`).
+
+#### Writing a multi-line code block
+
+Multi-line code blocks can be useful for displaying examples. They are done between two lines of three backticks as usual in Markdown:
+
+
+````
+```
+# first line of code
+# second line
+# etc
+```
+````
+
+#### Writing a return block
+
+The return block should be introduced with the `Returns:` prefix, followed by a line return and an indentation.
+The first line should be the type of the return, followed by a line return. No need to indent further for the elements
+building the return.
+
+Here's an example of a single value return:
+
+```
+    Returns:
+        `List[int]`: A list of integers in the range [0, 1] --- 1 for a special token, 0 for a sequence token.
+```
+
+Here's an example of tuple return, comprising several objects:
+
+```
+    Returns:
+        `tuple(torch.FloatTensor)` comprising various elements depending on the configuration ([`BertConfig`]) and inputs:
+        - ** loss** (*optional*, returned when `masked_lm_labels` is provided) `torch.FloatTensor` of shape `(1,)` --
+          Total loss as the sum of the masked language modeling loss and the next sequence prediction (classification) loss.
+        - **prediction_scores** (`torch.FloatTensor` of shape `(batch_size, sequence_length, config.vocab_size)`) --
+          Prediction scores of the language modeling head (scores for each vocabulary token before SoftMax).
+```
+
+#### Adding an image
+
+Due to the rapidly growing repository, it is important to make sure that no files that would significantly weigh down the repository are added. This includes images, videos and other non-text files. We prefer to leverage a hf.co hosted `dataset` like
+the ones hosted on [`hf-internal-testing`](https://huggingface.co/hf-internal-testing) in which to place these files and reference
+them by URL. We recommend putting them in the following dataset: [huggingface/documentation-images](https://huggingface.co/datasets/huggingface/documentation-images).
+If an external contribution, feel free to add the images to your PR and ask a Hugging Face member to migrate your images
+to this dataset.
+
+## Writing documentation examples
+
+The syntax for Example docstrings can look as follows:
+
+```
+    Example:
+
+    ```py
+    >>> from datasets import load_dataset
+    >>> ds = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="validation")
+    >>> def add_prefix(example):
+    ...     example["text"] = "Review: " + example["text"]
+    ...     return example
+    >>> ds = ds.map(add_prefix)
+    >>> ds[0:3]["text"]
+    ['Review: compassionately explores the seemingly irreconcilable situation between conservative christian parents and their estranged gay and lesbian children .',
+        'Review: the soundtrack alone is worth the price of admission .',
+        'Review: rodriguez does a splendid job of racial profiling hollywood style--casting excellent latin actors of all ages--a trend long overdue .']
+
+    # process a batch of examples
+    >>> ds = ds.map(lambda example: tokenizer(example["text"]), batched=True)
+    # set number of processors
+    >>> ds = ds.map(add_prefix, num_proc=4)
+    ```
+```
+
+The docstring should give a minimal, clear example of how the respective class or function is to be used in practice and also include the expected (ideally sensible) output.
+Often, readers will try out the example before even going through the function 
+or class definitions. Therefore, it is of utmost importance that the example 
+works as expected.

docs/source/_config.py

@@ -0,0 +1,11 @@
+# docstyle-ignore
+INSTALL_CONTENT = """
+# Datasets installation
+! pip install datasets transformers
+# To install from source instead of the last release, comment the command above and uncomment the following one.
+# ! pip install git+https://github.com/huggingface/datasets.git
+"""
+
+notebook_first_cells = [{"type": "code", "content": INSTALL_CONTENT}]
+default_branch_name = "main"
+version_prefix = ""

docs/source/_redirects.yml

@@ -0,0 +1,13 @@
+# This first_section was backported from nginx
+loading_datasets: loading
+share_dataset: share
+quicktour: quickstart
+dataset_streaming: stream
+torch_tensorflow: use_dataset
+splits: loading#slice-splits
+processing: process
+faiss_and_ea: faiss_es
+features: about_dataset_features
+exploring: access
+package_reference/logging_methods: package_reference/utilities
+# end of first_section

docs/source/_toctree.yml

@@ -0,0 +1,148 @@
+- sections: 
+  - local: index
+    title: 🤗 Datasets
+  - local: quickstart
+    title: Quickstart
+  - local: installation
+    title: Installation
+  title: Get started
+- sections:
+  - local: tutorial
+    title: Overview
+  - local: load_hub
+    title: Load a dataset from the Hub
+  - local: access
+    title: Know your dataset
+  - local: use_dataset
+    title: Preprocess
+  - local: create_dataset
+    title: Create a dataset
+  - local: upload_dataset
+    title: Share a dataset to the Hub
+  title: "Tutorials"
+- sections:
+  - local: how_to
+    title: Overview
+  - sections:
+    - local: loading
+      title: Load
+    - local: process
+      title: Process
+    - local: stream
+      title: Stream
+    - local: use_with_pytorch
+      title: Use with PyTorch
+    - local: use_with_tensorflow
+      title: Use with TensorFlow
+    - local: use_with_numpy
+      title: Use with NumPy
+    - local: use_with_jax
+      title: Use with JAX
+    - local: use_with_pandas
+      title: Use with Pandas
+    - local: use_with_polars
+      title: Use with Polars
+    - local: use_with_pyarrow
+      title: Use with PyArrow
+    - local: use_with_spark
+      title: Use with Spark
+    - local: cache
+      title: Cache management
+    - local: filesystems
+      title: Cloud storage
+    - local: faiss_es
+      title: Search index
+    - local: cli
+      title: CLI
+    - local: troubleshoot
+      title: Troubleshooting
+    title: "General usage"
+  - sections:
+    - local: audio_load
+      title: Load audio data
+    - local: audio_process
+      title: Process audio data
+    - local: audio_dataset
+      title: Create an audio dataset
+    title: "Audio"
+  - sections:
+    - local: image_load
+      title: Load image data
+    - local: image_process
+      title: Process image data
+    - local: image_dataset
+      title: Create an image dataset
+    - local: depth_estimation
+      title: Depth estimation
+    - local: image_classification
+      title: Image classification
+    - local: semantic_segmentation
+      title: Semantic segmentation
+    - local: object_detection
+      title: Object detection
+    - local: video_load
+      title: Load video data
+    - local: video_dataset
+      title: Create a video dataset
+    - local: document_load
+      title: Load document data
+    - local: document_dataset
+      title: Create a document dataset
+    - local: nifti_dataset
+      title: Create a medical imaging dataset
+    title: "Vision"
+  - sections:
+    - local: mesh_load
+      title: Load mesh data
+    - local: mesh_dataset
+      title: Create a mesh dataset
+    title: "3D"
+  - sections:
+    - local: nlp_load
+      title: Load text data
+    - local: nlp_process
+      title: Process text data
+    title: "Text"
+  - sections:
+    - local: tabular_load
+      title: Load tabular data
+    title: "Tabular"
+  - sections:
+    - local: tsfile_load
+      title: Load TsFile data
+    title: "Time-series"
+  - sections:
+    - local: share
+      title: Share
+    - local: dataset_card
+      title: Create a dataset card
+    - local: repository_structure
+      title: Structure your repository
+    title: "Dataset repository"
+  title: "How-to guides"
+- sections:
+  - local: about_arrow
+    title: Datasets 🤝 Arrow
+  - local: about_cache
+    title: The cache
+  - local: about_mapstyle_vs_iterable
+    title: Dataset or IterableDataset
+  - local: about_dataset_features
+    title: Dataset features
+  - local: about_dataset_load
+    title: Build and load
+  - local: about_map_batch
+    title: Batch mapping
+  title: "Conceptual guides"
+- sections:
+  - local: package_reference/main_classes
+    title: Main classes
+  - local: package_reference/builder_classes
+    title: Builder classes
+  - local: package_reference/loading_methods
+    title: Loading methods
+  - local: package_reference/table_classes
+    title: Table Classes
+  - local: package_reference/utilities
+    title: Utilities
+  title: "Reference"

docs/source/about_arrow.md

@@ -0,0 +1,50 @@
+# Datasets 🤝 Arrow
+
+## What is Arrow?
+
+[Arrow](https://arrow.apache.org/) enables large amounts of data to be processed and moved quickly. It is a specific data format that stores data in a columnar memory layout. This provides several significant advantages:
+
+* Arrow's standard format allows [zero-copy reads](https://en.wikipedia.org/wiki/Zero-copy) which removes virtually all serialization overhead.
+* Arrow is language-agnostic so it supports different programming languages.
+* Arrow is column-oriented so it is faster at querying and processing slices or columns of data.
+* Arrow allows for copy-free hand-offs to standard machine learning tools such as NumPy, Pandas, PyTorch, and TensorFlow.
+* Arrow supports many, possibly nested, column types.
+
+## Memory-mapping
+
+🤗 Datasets uses Arrow for its local caching system. It allows datasets to be backed by an on-disk cache, which is memory-mapped for fast lookup.
+This architecture allows for large datasets to be used on machines with relatively small device memory.
+
+For example, loading the full English Wikipedia dataset only takes a few MB of RAM:
+
+```python
+>>> import os; import psutil; import timeit
+>>> from datasets import load_dataset
+
+# Process.memory_info is expressed in bytes, so convert to megabytes 
+>>> mem_before = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024)
+>>> wiki = load_dataset("wikimedia/wikipedia", "20220301.en", split="train")
+>>> mem_after = psutil.Process(os.getpid()).memory_info().rss / (1024 * 1024)
+
+>>> print(f"RAM memory used: {(mem_after - mem_before)} MB")
+RAM memory used: 50 MB
+```
+
+This is possible because the Arrow data is actually memory-mapped from disk, and not loaded in memory.
+Memory-mapping allows access to data on disk, and leverages virtual memory capabilities for fast lookups.
+
+## Performance
+
+Iterating over a memory-mapped dataset using Arrow is fast. Iterating over Wikipedia on a laptop gives you speeds of 1-3 Gbit/s:
+
+```python
+>>> s = """batch_size = 1000
+... for batch in wiki.iter(batch_size):
+...     ...
+... """
+
+>>> elapsed_time = timeit.timeit(stmt=s, number=1, globals=globals())
+>>> print(f"Time to iterate over the {wiki.dataset_size >> 30} GB dataset: {elapsed_time:.1f} sec, "
+...       f"ie. {float(wiki.dataset_size >> 27)/elapsed_time:.1f} Gb/s")
+Time to iterate over the 18 GB dataset: 31.8 sec, ie. 4.8 Gb/s
+```

docs/source/about_cache.mdx

@@ -0,0 +1,49 @@
+# The cache
+
+The cache is one of the reasons why 🤗 Datasets is so efficient. It stores previously downloaded and processed datasets so when you need to use them again, they are reloaded directly from the cache. This avoids having to download a dataset all over again, or reapplying processing functions. Even after you close and start another Python session, 🤗 Datasets will reload your dataset directly from the cache!
+
+## Fingerprint
+
+How does the cache keeps track of what transforms are applied to a dataset? Well, 🤗 Datasets assigns a fingerprint to the cache file. A fingerprint keeps track of the current state of a dataset. The initial fingerprint is computed using a hash from the Arrow table, or a hash of the Arrow files if the dataset is on disk. Subsequent fingerprints are computed by combining the fingerprint of the previous state, and a hash of the latest transform applied. 
+
+> [!TIP]
+> Transforms are any of the processing methods from the [How-to Process](./process) guides such as [`Dataset.map`] or [`Dataset.shuffle`].
+
+Here are what the actual fingerprints look like:
+
+```py
+>>> from datasets import Dataset
+>>> dataset1 = Dataset.from_dict({"a": [0, 1, 2]})
+>>> dataset2 = dataset1.map(lambda x: {"a": x["a"] + 1})
+>>> print(dataset1._fingerprint, dataset2._fingerprint)
+d19493523d95e2dc 5b86abacd4b42434
+```
+
+In order for a transform to be hashable, it needs to be picklable by [dill](https://dill.readthedocs.io/en/latest/) or [pickle](https://docs.python.org/3/library/pickle). 
+
+When you use a non-hashable transform, 🤗 Datasets uses a random fingerprint instead and raises a warning. The non-hashable transform is considered different from the previous transforms. As a result, 🤗 Datasets will recompute all the transforms. Make sure your transforms are serializable with pickle or dill to avoid this!
+
+An example of when 🤗 Datasets recomputes everything is when caching is disabled. When this happens, the cache files are generated every time and they get written to a temporary directory. Once your Python session ends, the cache files in the temporary directory are deleted. A random hash is assigned to these cache files, instead of a fingerprint. 
+
+> [!TIP]
+> When caching is disabled, use [`Dataset.save_to_disk`] to save your transformed dataset or it will be deleted once the session ends.
+
+## Hashing
+
+The fingerprint of a dataset is updated by hashing the function passed to `map` as well as the `map` parameters (`batch_size`, `remove_columns`, etc.).
+
+You can check the hash of any Python object using the [`fingerprint.Hasher`]:
+
+```py
+>>> from datasets.fingerprint import Hasher
+>>> my_func = lambda example: {"length": len(example["text"])}
+>>> print(Hasher.hash(my_func))
+'3d35e2b3e94c81d6'
+```
+
+The hash is computed by dumping the object using a `dill` pickler and hashing the dumped bytes.
+The pickler recursively dumps all the variables used in your function, so any change you do to an object that is used in your function, will cause the hash to change.
+
+If one of your functions doesn't seem to have the same hash across sessions, it means at least one of its variables contains a Python object that is not deterministic.
+When this happens, feel free to hash any object you find suspicious to try to find the object that caused the hash to change.
+For example, if you use a list for which the order of its elements is not deterministic across sessions, then the hash won't be the same across sessions either.

docs/source/about_dataset_features.mdx

@@ -0,0 +1,230 @@
+# Dataset features
+
+[`Features`] defines the internal structure of a dataset. It is used to specify the underlying serialization format. What's more interesting to you though is that [`Features`] contains high-level information about everything from the column names and types, to the [`ClassLabel`]. You can think of [`Features`] as the backbone of a dataset.
+
+The [`Features`] format is simple: `dict[column_name, column_type]`. It is a dictionary of column name and column type pairs. The column type provides a wide range of options for describing the type of data you have.
+
+Let's have a look at the features of the MRPC dataset from the GLUE benchmark:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('nyu-mll/glue', 'mrpc', split='train')
+>>> dataset.features
+{'idx': Value('int32'),
+ 'label': ClassLabel(names=['not_equivalent', 'equivalent']),
+ 'sentence1': Value('string'),
+ 'sentence2': Value('string'),
+}
+```
+
+The [`Value`] feature tells 🤗 Datasets:
+
+- The `idx` data type is `int32`.
+- The `sentence1` and `sentence2` data types are `string`.
+
+🤗 Datasets supports many other data types such as `bool`, `float32` and `binary` to name just a few.
+
+> [!TIP]
+> Refer to [`Value`] for a full list of supported data types.
+
+The [`ClassLabel`] feature informs 🤗 Datasets the `label` column contains two classes. The classes are labeled `not_equivalent` and `equivalent`. Labels are stored as integers in the dataset. When you retrieve the labels, [`ClassLabel.int2str`] and [`ClassLabel.str2int`] carries out the conversion from integer value to label name, and vice versa.
+
+If your data type contains a list of objects, then you want to use the [`List`] feature. Remember the SQuAD dataset?
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('rajpurkar/squad', split='train')
+>>> dataset.features
+{'id': Value('string'),
+ 'title': Value('string'),
+ 'context': Value('string'),
+ 'question': Value('string'),
+ 'answers': {'text': List(Value('string')),
+  'answer_start': List(Value('int32'))}}
+```
+
+The `answers` field is constructed using the dict of features because and contains two subfields, `text` and `answer_start`, which are lists of `string` and `int32`, respectively.
+
+> [!TIP]
+> See the [flatten](./process#flatten) section to learn how you can extract the nested subfields as their own independent columns.
+
+The array feature type is useful for creating arrays of various sizes. You can create arrays with two dimensions using [`Array2D`], and even arrays with five dimensions using [`Array5D`].
+
+```py
+>>> features = Features({'a': Array2D(shape=(1, 3), dtype='int32')})
+```
+
+The array type also allows the first dimension of the array to be dynamic. This is useful for handling sequences with variable lengths such as sentences, without having to pad or truncate the input to a uniform shape.
+
+```py
+>>> features = Features({'a': Array3D(shape=(None, 5, 2), dtype='int32')})
+```
+
+## Audio feature
+
+Audio datasets have a column with type [`Audio`], which contains three important fields:
+
+- `array`: the decoded audio data represented as a 1-dimensional array.
+- `path`: the path to the downloaded audio file.
+- `sampling_rate`: the sampling rate of the audio data.
+
+When you load an audio dataset and call the audio column, the [`Audio`] feature automatically decodes and resamples the audio file:
+
+```py
+>>> from datasets import load_dataset, Audio
+
+>>> dataset = load_dataset("PolyAI/minds14", "en-US", split="train")
+>>> dataset[0]["audio"]
+<datasets.features._torchcodec.AudioDecoder object at 0x11642b6a0>
+```
+
+> [!WARNING]
+> Index into an audio dataset using the row index first and then the `audio` column - `dataset[0]["audio"]` - to avoid decoding and resampling all the audio files in the dataset. Otherwise, this can be a slow and time-consuming process if you have a large dataset.
+
+With `decode=False`, the [`Audio`] type simply gives you the path or the bytes of the audio file, without decoding it into an torchcodec `AudioDecoder` object,
+
+```py
+>>> dataset = load_dataset("PolyAI/minds14", "en-US", split="train").cast_column("audio", Audio(decode=False))
+>>> dataset[0]
+{'audio': {'bytes': None,
+  'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav'},
+ 'english_transcription': 'I would like to set up a joint account with my partner',
+ 'intent_class': 11,
+ 'lang_id': 4,
+ 'path': '/root/.cache/huggingface/datasets/downloads/extracted/f14948e0e84be638dd7943ac36518a4cf3324e8b7aa331c5ab11541518e9368c/en-US~JOINT_ACCOUNT/602ba55abb1e6d0fbce92065.wav',
+ 'transcription': 'I would like to set up a joint account with my partner'}
+```
+
+## Image feature
+
+Image datasets have a column with type [`Image`], which loads `PIL.Image` objects from images stored as bytes:
+
+When you load an image dataset and call the image column, the [`Image`] feature automatically decodes the image file:
+
+```py
+>>> from datasets import load_dataset, Image
+
+>>> dataset = load_dataset("AI-Lab-Makerere/beans", split="train")
+>>> dataset[0]["image"]
+<PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=500x500 at 0x125506CF8>
+```
+
+> [!WARNING]
+> Index into an image dataset using the row index first and then the `image` column - `dataset[0]["image"]` - to avoid decoding all the image files in the dataset. Otherwise, this can be a slow and time-consuming process if you have a large dataset.
+
+With `decode=False`, the [`Image`] type simply gives you the path or the bytes of the image file, without decoding it into an `PIL.Image`,
+
+```py
+>>> dataset = load_dataset("AI-Lab-Makerere/beans", split="train").cast_column("image", Image(decode=False))
+>>> dataset[0]["image"]
+{'bytes': None,
+ 'path': '/Users/username/.cache/huggingface/datasets/downloads/extracted/772e7c1fba622cff102b85dd74bcce46e8168634df4eaade7bedd3b8d91d3cd7/train/healthy/healthy_train.265.jpg'}
+```
+
+Depending on the dataset, you may get the path to the local downloaded image, or the content of the image as bytes if the dataset is not made of individual files.
+
+You can also define a dataset of images from numpy arrays:
+
+```python
+>>> ds = Dataset.from_dict({"i": [np.zeros(shape=(16, 16, 3), dtype=np.uint8)]}, features=Features({"i": Image()}))
+```
+
+And in this case the numpy arrays are encoded into PNG (or TIFF if the pixels values precision is important).
+
+For multi-channels arrays like RGB or RGBA, only uint8 is supported. If you use a larger precision, you get a warning and the array is downcasted to uint8.
+For gray-scale images you can use the integer or float precision you want as long as it is compatible with `Pillow`. A warning is shown if your image integer or float precision is too high, and in this case the array is downcated: an int64 array is downcasted to int32, and a float64 array is downcasted to float32.
+
+## Mesh feature
+
+Mesh datasets have a column with type [`Mesh`], which loads 3D mesh files as `trimesh` objects.
+
+When you load a mesh dataset and call the mesh column, the [`Mesh`] feature automatically decodes the mesh file:
+
+```py
+>>> from datasets import load_dataset, Mesh
+
+>>> dataset = load_dataset("VINAY-UMRETHE/My-Mesh-Dataset", split="train")
+>>> dataset[0]["mesh"]
+<trimesh.Scene(len(geometry)=33)>
+```
+
+Depending on the file content, `trimesh` may return a `trimesh.Trimesh` object or a `trimesh.Scene` object.
+
+> [!WARNING]
+> Index into a mesh dataset using the row index first and then the `mesh` column - `dataset[0]["mesh"]` - to avoid decoding all the mesh files in the dataset. Otherwise, this can be a slow and time-consuming process if you have a large dataset.
+
+With `decode=False`, the [`Mesh`] type gives you the path or bytes of the mesh file without decoding it with `trimesh`:
+
+```py
+>>> dataset = load_dataset("VINAY-UMRETHE/My-Mesh-Dataset", split="train").cast_column("mesh", Mesh(decode=False))
+>>> dataset[0]["mesh"]
+{'bytes': b'...',
+ 'path': '00001.glb'}
+```
+
+The [`Mesh`] feature supports `.glb`, `.ply`, and `.stl` files. For embedded bytes, the stored `path` is used to infer the mesh file type.
+
+## Json feature
+
+Datasets are based on Arrow which is a columnar format, and therefore they expect every example to have the same type and subtypes, and dictionaries to have the same keys and values types.
+Loading a dataset errors out when fields have mismatching types, and fills missing fields in dictionaries with None so all dictionaries have the same keys and value types.
+
+To avoid this and allow mixed-types without errors, you can use `on_mixed_types="use_json"` or specify `features=` with a [`Json`] type:
+
+```python
+>>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]})
+Traceback (most recent call last):
+  ...
+  File "pyarrow/error.pxi", line 92, in pyarrow.lib.check_status
+pyarrow.lib.ArrowInvalid: Could not convert 'foo' with type str: tried to convert to int64
+
+>>> features = Features({"a": Json()})
+>>> ds = Dataset.from_dict({"a": [0, "foo", {"subfield": "bar"}]}, features=features)
+>>> ds.features
+{'a': Json()}
+>>> list(ds["a"])
+[0, "foo", {"subfield": "bar"}]
+```
+
+This is also useful for lists of dictionaries with arbitrary keys and values, to avoid filling missing fields with None:
+
+```python
+>>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]})
+>>> ds.features
+{'a': List({'b': Value('int64'), 'c': Value('int64')})}
+>>> list(ds["a"])
+[[{'b': 0, 'c': None}, {'b': None, 'c': 0}]]  # missing fields are filled with None
+
+>>> features = Features({"a": List(Json())})
+>>> ds = Dataset.from_dict({"a": [[{"b": 0}, {"c": 0}]]}, features=features)
+>>> ds.features
+{'a': List(Json())}
+>>> list(ds["a"])
+[[{'b': 0}, {'c': 0}]]  # OK
+```
+
+Another example with tool calling data and the `on_mixed_types="use_json"` argument (useful to not have to specify `features=` manually):
+
+```python
+>>> messages = [
+...     {"role": "user", "content": "Turn on the living room lights and play my electronic music playlist."},
+...     {"role": "assistant", "tool_calls": [
+...         {"type": "function", "function": {
+...             "name": "control_light",
+...             "arguments": {"room": "living room", "state": "on"}
+...         }},
+...         {"type": "function", "function": {
+...             "name": "play_music",
+...             "arguments": {"playlist": "electronic"}  # mixed-type here since keys ["playlist"] and ["room", "state"] are different
+...         }}]
+...     },
+...     {"role": "tool", "name": "control_light", "content": "The lights in the living room are now on."},
+...     {"role": "tool", "name": "play_music", "content": "The music is now playing."},
+...     {"role": "assistant", "content": "Done!"}
+... ]
+>>> ds = Dataset.from_dict({"messages": [messages]}, on_mixed_types="use_json")
+>>> ds.features
+{'messages': List({'role': Value('string'), 'content': Value('string'), 'tool_calls': List(Json()), 'name': Value('string')})}
+>>> ds[0][1]["tool_calls"][0]["function"]["arguments"]
+{"room": "living room", "state": "on"}
+```

docs/source/about_dataset_load.mdx

@@ -0,0 +1,104 @@
+# Build and load
+
+Nearly every deep learning workflow begins with loading a dataset, which makes it one of the most important steps. With 🤗 Datasets, there are more than 900 datasets available to help you get started with your NLP task. All you have to do is call: [`load_dataset`] to take your first step. This function is a true workhorse in every sense because it builds and loads every dataset you use.
+
+## ELI5: `load_dataset`
+
+Let's begin with a basic Explain Like I'm Five.
+
+A dataset is a directory that contains:
+
+- Some data files in generic formats (JSON, CSV, Parquet, text, etc.)
+- A dataset card named `README.md` that contains documentation about the dataset as well as a YAML header to define the datasets tags and configurations
+
+The [`load_dataset`] function fetches the requested dataset locally or from the Hugging Face Hub.
+The Hub is a central repository where all the Hugging Face datasets and models are stored.
+
+If the dataset only contains data files, then [`load_dataset`] automatically infers how to load the data files from their extensions (json, csv, parquet, tsfile, txt, etc.).
+Under the hood, 🤗 Datasets will use an appropriate [`DatasetBuilder`] based on the data files format. There exist one builder per data file format in 🤗 Datasets:
+
+* [`datasets.packaged_modules.text.Text`] for text
+* [`datasets.packaged_modules.csv.Csv`] for CSV and TSV
+* [`datasets.packaged_modules.json.Json`] for JSON and JSONL
+* [`datasets.packaged_modules.parquet.Parquet`] for Parquet
+* [`datasets.packaged_modules.arrow.Arrow`] for Arrow (streaming file format)
+* [`datasets.packaged_modules.sql.Sql`] for SQL databases
+* [`datasets.packaged_modules.tsfile.TsFile`] for TsFile (time-series data)
+* [`datasets.packaged_modules.imagefolder.ImageFolder`] for image folders
+* [`datasets.packaged_modules.audiofolder.AudioFolder`] for audio folders
+
+> [!TIP]
+> Read the [Share](./upload_dataset) section to learn more about how to share a dataset.
+
+🤗 Datasets downloads the dataset files from the original URL, generates the dataset and caches it in an Arrow table on your drive.
+If you've downloaded the dataset before, then 🤗 Datasets will reload it from the cache to save you the trouble of downloading it again.
+
+Now that you have a high-level understanding about how datasets are built, let's take a closer look at the nuts and bolts of how all this works.
+
+## Building a dataset
+
+When you load a dataset for the first time, 🤗 Datasets takes the raw data file and builds it into a table of rows and typed columns. There are two main classes responsible for building a dataset: [`BuilderConfig`] and [`DatasetBuilder`].
+
+
+<div class="flex justify-center">
+   <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/builderconfig.png"/>
+</div>
+
+### BuilderConfig[[datasets-builderconfig]]
+
+[`BuilderConfig`] is the configuration class of [`DatasetBuilder`]. The [`BuilderConfig`] contains the following basic attributes about a dataset:
+
+| Attribute     | Description                                                  |
+|---------------|--------------------------------------------------------------|
+| `name`        | Short name of the dataset.                                   |
+| `version`     | Dataset version identifier.                                  |
+| `data_dir`    | Stores the path to a local folder containing the data files. |
+| `data_files`  | Stores paths to local data files.                            |
+| `description` | Description of the dataset.                                  |
+
+If you want to add additional attributes to your dataset such as the class labels, you can subclass the base [`BuilderConfig`] class. There are two ways to populate the attributes of a [`BuilderConfig`] class or subclass:
+
+- Provide a list of predefined [`BuilderConfig`] class (or subclass) instances in the datasets [`DatasetBuilder.BUILDER_CONFIGS`] attribute.
+
+- When you call [`load_dataset`], any keyword arguments that are not specific to the method will be used to set the associated attributes of the [`BuilderConfig`] class. This will override the predefined attributes if a specific configuration was selected.
+
+You can also set the [`DatasetBuilder.BUILDER_CONFIG_CLASS`] to any custom subclass of [`BuilderConfig`].
+
+### DatasetBuilder[[datasets-datasetbuilder]]
+
+[`DatasetBuilder`] accesses all the attributes inside [`BuilderConfig`] to build the actual dataset.
+
+<div class="flex justify-center">
+   <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/datasetbuilder.png"/>
+</div>
+
+There are three main methods in [`DatasetBuilder`]:
+
+1. [`DatasetBuilder._info`] is in charge of defining the dataset attributes. When you call `dataset.info`, 🤗 Datasets returns the information stored here. Likewise, the [`Features`] are also specified here. Remember, the [`Features`] are like the skeleton of the dataset. It provides the names and types of each column.
+
+2. [`DatasetBuilder._split_generator`] downloads or retrieves the requested data files, organizes them into splits, and defines specific arguments for the generation process. This method has a [`DownloadManager`] that downloads files or fetches them from your local filesystem. Within the [`DownloadManager`], there is a [`DownloadManager.download_and_extract`] method that accepts a dictionary of URLs to the original data files, and downloads the requested files. Accepted inputs include: a single URL or path, or a list/dictionary of URLs or paths. Any compressed file types like TAR, GZIP and ZIP archives will be automatically extracted.
+
+   Once the files are downloaded, [`SplitGenerator`] organizes them into splits. The [`SplitGenerator`] contains the name of the split, and any keyword arguments that are provided to the [`DatasetBuilder._generate_examples`] method. The keyword arguments can be specific to each split, and typically comprise at least the local path to the data files for each split.
+
+3. [`DatasetBuilder._generate_examples`] reads and parses the data files for a split. Then it yields dataset examples according to the format specified in the `features` from [`DatasetBuilder._info`]. The input of [`DatasetBuilder._generate_examples`] is actually the `filepath` provided in the keyword arguments of the last method.
+
+   The dataset is generated with a Python generator, which doesn't load all the data in memory. As a result, the generator can handle large datasets. However, before the generated samples are flushed to the dataset file on disk, they are stored in an `ArrowWriter` buffer. This means the generated samples are written by batch. If your dataset samples consumes a lot of memory (images or videos), then make sure to specify a low value for the `DEFAULT_WRITER_BATCH_SIZE` attribute in [`DatasetBuilder`]. We recommend not exceeding a size of 200 MB.
+
+## Maintaining integrity
+
+To ensure a dataset is complete, [`load_dataset`] will perform a series of tests on the downloaded files to make sure everything is there. This way, you don't encounter any surprises when your requested dataset doesn't get generated as expected. [`load_dataset`] verifies:
+
+- The number of splits in the generated `DatasetDict`.
+- The number of samples in each split of the generated `DatasetDict`.
+- The list of downloaded files.
+- The SHA256 checksums of the downloaded files (disabled by default).
+
+If the dataset doesn't pass the verifications, it is likely that the dataset author made some changes in the data files.
+
+In this case, an error is raised to alert that the dataset has changed.
+To ignore the error, one needs to specify `verification_mode="no_checks"` in [`load_dataset`].
+Anytime you see a verification error, feel free to open a discussion or pull request in the corresponding dataset "Community" tab, so that the integrity checks for that dataset are updated.
+
+## Security
+
+The dataset repositories on the Hub are scanned for malware, see more information [here](https://huggingface.co/docs/hub/security#malware-scanning).

docs/source/about_map_batch.mdx

@@ -0,0 +1,59 @@
+# Batch mapping
+
+Combining the utility of [`Dataset.map`] with batch mode is very powerful. It allows you to speed up processing, and freely control the size of the generated dataset. 
+
+## Need for speed
+
+The primary objective of batch mapping is to speed up processing. Often times, it is faster to work with batches of data instead of single examples. Naturally, batch mapping lends itself to tokenization. For example, the 🤗 [Tokenizers](https://huggingface.co/docs/tokenizers/python/latest/) library works faster with batches because it parallelizes the tokenization of all the examples in a batch.
+
+## Input size != output size
+
+The ability to control the size of the generated dataset can be leveraged for many interesting use-cases. In the How-to [map](#map) section, there are examples of using batch mapping to:
+
+- Split long sentences into shorter chunks.
+- Augment a dataset with additional tokens.
+
+It is helpful to understand how this works, so you can come up with your own ways to use batch mapping. At this point, you may be wondering how you can control the size of the generated dataset. The answer is: **the mapped function does not have to return an output batch of the same size**.
+
+In other words, your mapped function input can be a batch of size `N` and return a batch of size `M`. The output `M` can be greater than or less than `N`. This means you can concatenate your examples, divide it up, and even add more examples!
+
+However, remember that all values in the output dictionary must contain the **same number of elements** as the other fields in the output dictionary. Otherwise, it is not possible to define the number of examples in the output returned by the mapped function. The number can vary between successive batches processed by the mapped function. For a single batch though, all values of the output dictionary should have the same length (i.e., the number of elements).
+
+For example, from a dataset of 1 column and 3 rows, if you use `map` to return a new column with twice as many rows, then you will have an error.
+In this case, you end up with one column with 3 rows, and one column with 6 rows. As you can see, the table will not be valid:
+
+```py
+>>> from datasets import Dataset
+>>> dataset = Dataset.from_dict({"a": [0, 1, 2]})
+>>> dataset.map(lambda batch: {"b": batch["a"] * 2}, batched=True)  # new column with 6 elements: [0, 1, 2, 0, 1, 2]
+'ArrowInvalid: Column 1 named b expected length 3 but got length 6'
+```
+
+To make it valid, you have to drop one of the columns:
+
+```py
+>>> from datasets import Dataset
+>>> dataset = Dataset.from_dict({"a": [0, 1, 2]})
+>>> dataset_with_duplicates = dataset.map(lambda batch: {"b": batch["a"] * 2}, remove_columns=["a"], batched=True)
+>>> len(dataset_with_duplicates)
+6
+```
+Alternatively, you can overwrite the existing column to achieve the same result.
+For example, here’s how to duplicate every row in the dataset by overwriting column `"a"`:
+
+```py
+>>> from datasets import Dataset
+>>> dataset = Dataset.from_dict({"a": [0, 1, 2]})
+# overwrites the existing "a" column with duplicated values
+>>> duplicated_dataset = dataset.map(
+...     lambda batch: {"a": [x for x in batch["a"] for _ in range(2)]},
+...     batched=True
+... )
+>>> duplicated_dataset
+Dataset({
+    features: ['a'],
+    num_rows: 6
+})
+>>> duplicated_dataset["a"]
+[0, 0, 1, 1, 2, 2]
+```

docs/source/about_mapstyle_vs_iterable.mdx

@@ -0,0 +1,250 @@
+# Differences between Dataset and IterableDataset
+
+There are two types of dataset objects, a [`Dataset`] and an [`IterableDataset`].
+Whichever type of dataset you choose to use or create depends on the size of the dataset.
+In general, an [`IterableDataset`] is ideal for big datasets (think hundreds of GBs!) due to its lazy behavior and speed advantages, while a [`Dataset`] is great for everything else.
+This page will compare the differences between a [`Dataset`] and an [`IterableDataset`] to help you pick the right dataset object for you.
+
+## Downloading and streaming
+
+When you have a regular [`Dataset`], you can access it using `my_dataset[0]`. This provides random access to the rows.
+Such datasets are also called "map-style" datasets.
+For example you can download ImageNet-1k like this and access any row:
+
+```python
+from datasets import load_dataset
+
+imagenet = load_dataset("timm/imagenet-1k-wds", split="train")  # downloads the full dataset
+print(imagenet[0])
+```
+
+But one caveat is that you must have the entire dataset stored on your disk or in memory, which blocks you from accessing datasets bigger than the disk.
+Because it can become inconvenient for big datasets, there exists another type of dataset, the [`IterableDataset`].
+When you have an `IterableDataset`, you can access it using a `for` loop to load the data progressively as you iterate over the dataset.
+This way, only a small fraction of examples is loaded in memory, and you don't write anything on disk.
+
+For example, you can stream the ImageNet-1k dataset without downloading it on disk:
+
+```python
+from datasets import load_dataset
+
+imagenet = load_dataset("timm/imagenet-1k-wds", split="train", streaming=True)  # will start loading the data when iterated over
+for example in imagenet:
+    print(example)
+    break
+```
+
+Streaming can read online data without writing any file to disk.
+For example, you can stream datasets made out of multiple shards, each of which is hundreds of gigabytes like [C4](https://huggingface.co/datasets/c4)  or [LAION-2B](https://huggingface.co/datasets/laion/laion2B-en).
+Learn more about how to stream a dataset in the [Dataset Streaming Guide](./stream).
+
+This is not the only difference though, because the "lazy" behavior of an `IterableDataset` is also present when it comes to dataset creation and processing.
+
+## Creating map-style datasets and iterable datasets
+
+You can create a [`Dataset`] using lists or dictionaries, and the data is entirely converted to Arrow so you can easily access any row:
+```python
+my_dataset = Dataset.from_dict({"col_1": [0, 1, 2, 3, 4, 5, 6, 7, 8, 9]})
+print(my_dataset[0])
+```
+
+To create an `IterableDataset` on the other hand, you must provide a "lazy" way to load the data.
+In Python, we generally use generator functions. These functions `yield` one example at a time, which means you can't access a row by slicing it like a regular `Dataset`:
+```python
+def my_generator(n):
+    for i in range(n):
+        yield {"col_1": i}
+
+my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs={"n": 10})
+for example in my_iterable_dataset:
+    print(example)
+    break
+```
+
+## Loading local files entirely and progressively
+
+It is possible to convert local or remote data files to an Arrow [`Dataset`] using [`load_dataset`]:
+```python
+data_files = {"train": ["path/to/data.csv"]}
+my_dataset = load_dataset("csv", data_files=data_files, split="train")
+print(my_dataset[0])
+```
+
+However, this requires a conversion step from CSV to Arrow format, which takes time and disk space if your dataset is big.
+
+To save disk space and skip the conversion step, you can define an `IterableDataset` by streaming from the local files directly.
+This way, the data is read progressively from the local files as you iterate over the dataset:
+
+```python
+data_files = {"train": ["path/to/data.csv"]}
+my_iterable_dataset = load_dataset("csv", data_files=data_files, split="train", streaming=True)
+for example in my_iterable_dataset:  # this reads the CSV file progressively as you iterate over the dataset
+    print(example)
+    break
+```
+
+Many file formats are supported, like CSV, JSONL, and Parquet, as well as image and audio files.
+You can find more information in the corresponding guides for loading [tabular](./tabular_load), [text](./nlp_load), [vision](./image_load), and [audio](./audio_load]) datasets.
+
+## Eager data processing and lazy data processing
+
+When you process a [`Dataset`] object using [`Dataset.map`], the entire dataset is processed immediately and returned.
+This is similar to how `pandas` works for example.
+
+```python
+my_dataset = my_dataset.map(process_fn)  # process_fn is applied on all the examples of the dataset
+print(my_dataset[0])
+```
+
+On the other hand, due to the "lazy" nature of an `IterableDataset`, calling [`IterableDataset.map`] does not apply your `map` function over the full dataset.
+Instead, your `map` function is applied on-the-fly.
+
+Because of that, you can chain multiple processing steps and they will all run at once when you start iterating over the dataset:
+
+```python
+my_iterable_dataset = my_iterable_dataset.map(process_fn_1)
+my_iterable_dataset = my_iterable_dataset.filter(filter_fn)
+my_iterable_dataset = my_iterable_dataset.map(process_fn_2)
+
+# process_fn_1, filter_fn and process_fn_2 are applied on-the-fly when iterating over the dataset
+for example in my_iterable_dataset:  
+    print(example)
+    break
+```
+
+## Exact and fast approximate shuffling
+
+When you shuffle a [`Dataset`] using [`Dataset.shuffle`], you apply an exact shuffling of the dataset.
+It works by taking a list of indices `[0, 1, 2, ... len(my_dataset) - 1]` and shuffling this list.
+Then, accessing `my_dataset[0]` returns the row and index defined by the first element of the indices mapping that has been shuffled:
+```python
+my_dataset = my_dataset.shuffle(seed=42)
+print(my_dataset[0])
+```
+
+Since we don't have random access to the rows in the case of an `IterableDataset`, we can't use a shuffled list of indices and access a row at an arbitrary position.
+This prevents the use of exact shuffling.
+Instead, a fast approximate shuffling is used in [`IterableDataset.shuffle`].
+It uses a shuffle buffer to sample random examples iteratively from the dataset.
+Since the dataset is still read iteratively, it provides excellent speed performance:
+```python
+my_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100)
+for example in my_iterable_dataset:
+    print(example)
+    break
+```
+
+But using a shuffle buffer is not enough to provide a satisfactory shuffling for machine learning model training. So [`IterableDataset.shuffle`] also shuffles the dataset shards if your dataset is made of multiple files or sources. Additionally, it fills the buffer using up to `max_buffer_input_shards` shards at a time (default is 10), which greatly improves the quality of the shuffling:
+
+```python
+# Stream from the internet
+my_iterable_dataset = load_dataset("deepmind/code_contests", split="train", streaming=True)
+my_iterable_dataset.num_shards  # 39
+
+# Stream from local files
+data_files = {"train": [f"path/to/data_{i}.csv" for i in range(1024)]}
+my_iterable_dataset = load_dataset("csv", data_files=data_files, split="train", streaming=True)
+my_iterable_dataset.num_shards  # 1024
+
+# From a generator function
+def my_generator(n, sources):
+    for source in sources:
+        for example_id_for_current_source in range(n):
+            yield {"example_id": f"{source}_{example_id_for_current_source}"}
+
+gen_kwargs = {"n": 10, "sources": [f"path/to/data_{i}" for i in range(1024)]}
+my_iterable_dataset = IterableDataset.from_generator(my_generator, gen_kwargs=gen_kwargs)
+my_iterable_dataset.num_shards  # 1024
+```
+
+> [!TIP]
+>
+> If your dataset has few files, you can improve shuffling quality by calling [`~IterableDataset.reshard`] before shuffling. For example, for Parquet datasets this reshards using row groups instead of having one file per shard, giving you many more shards to shuffle from.
+
+## Speed differences
+
+Regular [`Dataset`] objects are based on Arrow which provides fast random access to the rows.
+Thanks to memory mapping and the fact that Arrow is an in-memory format, reading data from disk doesn't do expensive system calls and deserialization.
+It provides even faster data loading when iterating using a `for` loop by iterating on contiguous Arrow record batches.
+
+However as soon as your [`Dataset`] has an indices mapping (via [`Dataset.shuffle`] for example), the speed can become 10x slower.
+This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren't reading contiguous chunks of data anymore.
+To restore the speed, you'd need to rewrite the entire dataset on your disk again using [`Dataset.flatten_indices`], which removes the indices mapping.
+This may take a lot of time depending on the size of your dataset though:
+
+```python
+my_dataset[0]  # fast
+my_dataset = my_dataset.shuffle(seed=42)
+my_dataset[0]  # up to 10x slower
+my_dataset = my_dataset.flatten_indices()  # rewrite the shuffled dataset on disk as contiguous chunks of data
+my_dataset[0]  # fast again
+```
+
+
+In this case, we recommend switching to an [`IterableDataset`] and leveraging its fast approximate shuffling method [`IterableDataset.shuffle`].
+It only shuffles the shards order and adds a shuffle buffer to your dataset, which keeps the speed of your dataset optimal.
+You can also reshuffle the dataset easily:
+
+```python
+for example in enumerate(my_iterable_dataset):  # fast
+    pass
+
+shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=42, buffer_size=100)
+
+for example in enumerate(shuffled_iterable_dataset):  # as fast as before
+    pass
+
+shuffled_iterable_dataset = my_iterable_dataset.shuffle(seed=1337, buffer_size=100)  # reshuffling using another seed is instantaneous
+
+for example in enumerate(shuffled_iterable_dataset):  # still as fast as before
+    pass
+```
+
+If you're using your dataset on multiple epochs, the effective seed to shuffle the shards order in the shuffle buffer is `seed + epoch`.
+It makes it easy to reshuffle a dataset between epochs:
+```python
+for epoch in range(n_epochs):
+    my_iterable_dataset.set_epoch(epoch)
+    for example in my_iterable_dataset:  # fast + reshuffled at each epoch using `effective_seed = seed + epoch`
+        pass
+```
+
+To restart the iteration of a map-style dataset, you can simply skip the first examples:
+
+```python
+my_dataset = my_dataset.select(range(start_index, len(dataset)))
+```
+
+But if you use a `DataLoader` with a `Sampler`, you should instead save the state of your sampler (you might have written a custom sampler that allows resuming).
+
+On the other hand, iterable datasets don't provide random access to a specific example index to resume from. But you can use [`IterableDataset.state_dict`] and [`IterableDataset.load_state_dict`] to resume from a checkpoint instead, similarly to what you can do for models and optimizers:
+
+```python
+>>> iterable_dataset = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3)
+>>> # save in the middle of training
+>>> state_dict = iterable_dataset.state_dict()
+>>> # and resume later
+>>> iterable_dataset.load_state_dict(state_dict)
+```
+
+Under the hood, the iterable dataset keeps track of the current shard being read and the example index in the current shard and it stores this info in the `state_dict`.
+
+To resume from a checkpoint, the dataset skips all the shards that were previously read to restart from the current shard. 
+Then it reads the shard and skips examples until it reaches the exact example from the checkpoint.
+
+Therefore restarting a dataset is quite fast, since it will not re-read the shards that have already been iterated on. Still, resuming a dataset is generally not instantaneous since it has to restart reading from the beginning of the current shard and skip examples until it reaches the checkpoint location.
+
+This can be used with the `StatefulDataLoader` from `torchdata`, see [streaming with a PyTorch DataLoader](./use_with_pytorch#stream-data).
+
+## Switch from map-style to iterable
+
+If you want to benefit from the "lazy" behavior of an [`IterableDataset`] or their speed advantages, you can switch your map-style [`Dataset`] to an [`IterableDataset`]:
+```python
+my_iterable_dataset = my_dataset.to_iterable_dataset()
+```
+
+If you want to shuffle your dataset or [use it with a PyTorch DataLoader](./use_with_pytorch#stream-data), we recommend generating a sharded [`IterableDataset`]:
+```python
+my_iterable_dataset = my_dataset.to_iterable_dataset(num_shards=1024)
+my_iterable_dataset.num_shards  # 1024
+```

docs/source/access.mdx

@@ -0,0 +1,163 @@
+# Know your dataset
+
+There are two types of dataset objects, a regular [`Dataset`] and then an ✨ [`IterableDataset`] ✨. A [`Dataset`] provides fast random access to the rows, and memory-mapping so that loading even large datasets only uses a relatively small amount of device memory. But for really, really big datasets that won't even fit on disk or in memory, an [`IterableDataset`] allows you to access and use the dataset without waiting for it to download completely!
+
+This tutorial will show you how to load and access a [`Dataset`] and an [`IterableDataset`].
+
+## Dataset
+
+When you load a dataset split, you'll get a [`Dataset`] object. You can do many things with a [`Dataset`] object, which is why it's important to learn how to manipulate and interact with the data stored inside. 
+ 
+This tutorial uses the [rotten_tomatoes](https://huggingface.co/datasets/rotten_tomatoes) dataset, but feel free to load any dataset you'd like and follow along!
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+```
+
+### Indexing
+
+A [`Dataset`] contains columns of data, and each column can be a different type of data. The *index*, or axis label, is used to access examples from the dataset. For example, indexing by the row returns a dictionary of an example from the dataset:
+
+```py
+# Get the first row in the dataset
+>>> dataset[0]
+{'label': 1,
+ 'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'}
+```
+
+Use the `-` operator to start from the end of the dataset:
+
+```py
+# Get the last row in the dataset
+>>> dataset[-1]
+{'label': 0,
+ 'text': 'things really get weird , though not particularly scary : the movie is all portent and no content .'}
+```
+
+Indexing by the column name returns a list of all the values in the column:
+
+```py
+>>> dataset["text"]
+['the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+ 'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .',
+ 'effective but too-tepid biopic',
+ ...,
+ 'things really get weird , though not particularly scary : the movie is all portent and no content .']
+```
+
+You can combine row and column name indexing to return a specific value at a position:
+
+```py
+>>> dataset[0]["text"]
+'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .'
+```
+
+Indexing order doesn't matter. Indexing by the column name first returns a [`Column`] object that you can index as usual with row indices:
+
+```py
+>>> import time
+
+>>> start_time = time.time()
+>>> text = dataset[0]["text"]
+>>> end_time = time.time()
+>>> print(f"Elapsed time: {end_time - start_time:.4f} seconds")
+Elapsed time: 0.0031 seconds
+
+>>> start_time = time.time()
+>>> text = dataset["text"][0]
+>>> end_time = time.time()
+>>> print(f"Elapsed time: {end_time - start_time:.4f} seconds")
+Elapsed time: 0.0042 seconds
+```
+
+### Slicing
+
+Slicing returns a slice - or subset - of the dataset, which is useful for viewing several rows at once. To slice a dataset, use the `:` operator to specify a range of positions. 
+
+```py
+# Get the first three rows
+>>> dataset[:3]
+{'label': [1, 1, 1],
+ 'text': ['the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+  'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .',
+  'effective but too-tepid biopic']}
+
+# Get rows between three and six
+>>> dataset[3:6]
+{'label': [1, 1, 1],
+ 'text': ['if you sometimes like to go to the movies to have fun , wasabi is a good place to start .',
+  "emerges as something rare , an issue movie that's so honest and keenly observed that it doesn't feel like one .",
+  'the film provides some great insight into the neurotic mindset of all comics -- even those who have reached the absolute top of the game .']}
+```
+
+## IterableDataset
+
+An [`IterableDataset`] is loaded when you set the `streaming` parameter to `True` in [`~datasets.load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> iterable_dataset = load_dataset("ethz/food101", split="train", streaming=True)
+>>> for example in iterable_dataset:
+...     print(example)
+...     break
+{'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at 0x7F0681F5C520>, 'label': 6}
+```
+
+You can also create an [`IterableDataset`] from an *existing* [`Dataset`], but it is faster than streaming mode because the dataset is streamed from local files:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+>>> iterable_dataset = dataset.to_iterable_dataset()
+```
+
+An [`IterableDataset`] progressively iterates over a dataset one example at a time, so you don't have to wait for the whole dataset to download before you can use it. As you can imagine, this is quite useful for large datasets you want to use immediately!
+
+### Indexing
+
+An [`IterableDataset`]'s behavior is different from a regular [`Dataset`]. You don't get random access to examples in an [`IterableDataset`]. Instead, you should iterate over its elements, for example, by calling `next(iter())` or with a `for` loop to return the next item from the [`IterableDataset`]:
+
+```py
+>>> next(iter(iterable_dataset))
+{'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at 0x7F0681F59B50>,
+ 'label': 6}
+
+>>> for example in iterable_dataset:
+...     print(example)
+...     break
+{'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at 0x7F7479DE82B0>, 'label': 6}
+```
+
+But an [`IterableDataset`] supports column indexing that returns an iterable for the column values:
+
+```py
+>>> next(iter(iterable_dataset["label"]))
+6
+```
+
+### Creating a subset
+
+You can return a subset of the dataset with a specific number of examples in it with [`IterableDataset.take`]:
+
+```py
+# Get first three examples
+>>> list(iterable_dataset.take(3))
+[{'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=384x512 at 0x7F7479DEE9D0>,
+  'label': 6},
+ {'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=512x512 at 0x7F7479DE8190>,
+  'label': 6},
+ {'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=512x383 at 0x7F7479DE8310>,
+  'label': 6}]
+```
+
+But unlike [slicing](access/#slicing), [`IterableDataset.take`] creates a new [`IterableDataset`]. 
+
+## Next steps
+
+Interested in learning more about the differences between these two types of datasets? Learn more about them in the [Differences between `Dataset` and `IterableDataset`](about_mapstyle_vs_iterable) conceptual guide.
+
+To get more hands-on with these datasets types, check out the [Process](process) guide to learn how to preprocess a [`Dataset`] or the [Stream](stream) guide to learn how to preprocess an [`IterableDataset`].

docs/source/audio_dataset.mdx

@@ -0,0 +1,192 @@
+# Create an audio dataset
+
+You can share a dataset with your team or with anyone in the community by creating a dataset repository on the Hugging Face Hub:
+
+```py
+from datasets import load_dataset
+
+dataset = load_dataset("<username>/my_dataset")
+```
+
+There are several methods for creating and sharing an audio dataset:
+
+- Create an audio dataset from local files in python with [`Dataset.push_to_hub`]. This is an easy way that requires only a few steps in python.
+
+- Create an audio dataset repository with the `AudioFolder` builder. This is a no-code solution for quickly creating an audio dataset with several thousand audio files.
+
+> [!TIP]
+> You can control access to your dataset by requiring users to share their contact information first. Check out the [Gated datasets](https://huggingface.co/docs/hub/datasets-gated) guide for more information about how to enable this feature on the Hub.
+
+## Local files
+
+You can load your own dataset using the paths to your audio files. Use the [`~Dataset.cast_column`] function to take a column of audio file paths, and cast it to the [`Audio`] feature:
+
+```py
+>>> audio_dataset = Dataset.from_dict({"audio": ["path/to/audio_1", "path/to/audio_2", ..., "path/to/audio_n"]}).cast_column("audio", Audio())
+>>> audio_dataset[0]["audio"]
+<datasets.features._torchcodec.AudioDecoder object at 0x11642b6a0>
+```
+
+Then upload the dataset to the Hugging Face Hub using [`Dataset.push_to_hub`]:
+
+```py
+audio_dataset.push_to_hub("<username>/my_dataset")
+```
+
+This will create a dataset repository containing your audio dataset:
+
+```
+my_dataset/
+├── README.md
+└── data/
+    └── train-00000-of-00001.parquet
+```
+
+## AudioFolder
+
+The `AudioFolder` is a dataset builder designed to quickly load an audio dataset with several thousand audio files without requiring you to write any code.
+
+> [!TIP]
+> 💡 Take a look at the [Split pattern hierarchy](repository_structure#split-pattern-hierarchy) to learn more about how `AudioFolder` creates dataset splits based on your dataset repository structure.
+
+`AudioFolder` automatically infers the class labels of your dataset based on the directory name. Store your dataset in a directory structure like:
+
+```
+folder/train/dog/golden_retriever.mp3
+folder/train/dog/german_shepherd.mp3
+folder/train/dog/chihuahua.mp3
+
+folder/train/cat/maine_coon.mp3
+folder/train/cat/bengal.mp3
+folder/train/cat/birman.mp3
+```
+
+If the dataset follows the `AudioFolder` structure, then you can load it directly with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_name")
+```
+
+This is equivalent to passing `audiofolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("audiofolder", data_dir="/path/to/folder")
+```
+
+You can also use `audiofolder` to load datasets involving multiple splits. To do so, your dataset directory should have the following structure:
+
+```
+folder/train/dog/golden_retriever.mp3
+folder/train/cat/maine_coon.mp3
+folder/test/dog/german_shepherd.mp3
+folder/test/cat/bengal.mp3
+```
+
+> [!WARNING]
+> If all audio files are contained in a single directory or if they are not on the same level of directory structure, `label` column won't be added automatically. If you need it, set `drop_labels=False` explicitly.
+
+If there is additional information you'd like to include about your dataset, like text captions or bounding boxes, add it as a `metadata.csv` file in your folder. This lets you quickly create datasets for different computer vision tasks like text captioning or object detection. You can also use a JSONL file `metadata.jsonl` or a Parquet file `metadata.parquet`.
+
+```
+folder/train/metadata.csv
+folder/train/0001.mp3
+folder/train/0002.mp3
+folder/train/0003.mp3
+```
+
+You can also zip your audio files, and in this case each zip should contain both the audio files and the metadata
+
+```
+folder/train.zip
+folder/test.zip
+folder/validation.zip
+```
+
+Your `metadata.csv` file must have a `file_name` or `*_file_name` field which links audio files with their metadata:
+
+```csv
+file_name,additional_feature
+0001.mp3,This is a first value of a text feature you added to your audio files
+0002.mp3,This is a second value of a text feature you added to your audio files
+0003.mp3,This is a third value of a text feature you added to your audio files
+```
+
+or using `metadata.jsonl`:
+
+```jsonl
+{"file_name": "0001.mp3", "additional_feature": "This is a first value of a text feature you added to your audio files"}
+{"file_name": "0002.mp3", "additional_feature": "This is a second value of a text feature you added to your audio files"}
+{"file_name": "0003.mp3", "additional_feature": "This is a third value of a text feature you added to your audio files"}
+```
+
+Here the `file_name` must be the name of the audio file next to the metadata file. More generally, it must be the relative path from the directory containing the metadata to the audio file.
+
+It's possible to point to more than one audio in each row in your dataset, for example if both your input and output are audio files:
+
+```jsonl
+{"input_file_name": "0001.mp3", "output_file_name": "0001_output.mp3"}
+{"input_file_name": "0002.mp3", "output_file_name": "0002_output.mp3"}
+{"input_file_name": "0003.mp3", "output_file_name": "0003_output.mp3"}
+```
+
+You can also define lists of audio files. In that case you need to name the field `file_names` or `*_file_names`. Here is an example:
+
+```jsonl
+{"recordings_file_names": ["0001_r0.mp3", "0001_r1.mp3"], label: "same_person"}
+{"recordings_file_names": ["0002_r0.mp3", "0002_r1.mp3"], label: "same_person"}
+{"recordings_file_names": ["0003_r0.mp3", "0003_r1.mp3"], label: "different_person"}
+```
+
+## WebDataset
+
+The [WebDataset](https://github.com/webdataset/webdataset) format is based on TAR archives and is suitable for big audio datasets.
+Indeed you can group your audio files in TAR archives (e.g. 1GB of audio files per TAR archive) and have thousands of TAR archives:
+
+```
+folder/train/00000.tar
+folder/train/00001.tar
+folder/train/00002.tar
+...
+```
+
+In the archives, each example is made of files sharing the same prefix:
+
+```
+e39871fd9fd74f55.mp3
+e39871fd9fd74f55.json
+f18b91585c4d3f3e.mp3
+f18b91585c4d3f3e.json
+ede6e66b2fb59aab.mp3
+ede6e66b2fb59aab.json
+ed600d57fcee4f94.mp3
+ed600d57fcee4f94.json
+...
+```
+
+You can put your audio files labels/captions/bounding boxes using JSON or text files for example.
+
+Load your WebDataset and it will create on column per file suffix (here "mp3" and "json"):
+
+```python
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("webdataset", data_dir="/path/to/folder", split="train")
+>>> dataset[0]["json"]
+{"transcript": "Hello there !", "speaker": "Obi-Wan Kenobi"}
+```
+
+It's also possible to have several audio files per example like this:
+
+```
+e39871fd9fd74f55.input.mp3
+e39871fd9fd74f55.output.mp3
+e39871fd9fd74f55.json
+f18b91585c4d3f3e.input.mp3
+f18b91585c4d3f3e.output.mp3
+f18b91585c4d3f3e.json
+...
+```
+
+For more details on the WebDataset format and the python library, please check the [WebDataset documentation](https://webdataset.github.io/webdataset).

docs/source/audio_load.mdx

@@ -0,0 +1,115 @@
+# Load audio data
+
+You can load an audio dataset using the [`Audio`] feature that automatically decodes and resamples the audio files when you access the examples.
+Audio decoding is based on the [`torchcodec`](https://github.com/pytorch/torchcodec) python package, which uses the [`FFmpeg`](https://www.ffmpeg.org/) C library under the hood.
+
+## Installation
+
+To work with audio datasets, you need to have the `audio` dependencies installed.
+Check out the [installation](./installation#audio) guide to learn how to install it.
+
+## Local files
+
+You can load your own dataset using the paths to your audio files. Use the [`~Dataset.cast_column`] function to take a column of audio file paths, and cast it to the [`Audio`] feature:
+
+```py
+>>> audio_dataset = Dataset.from_dict({"audio": ["path/to/audio_1", "path/to/audio_2", ..., "path/to/audio_n"]}).cast_column("audio", Audio())
+>>> audio_dataset[0]["audio"]
+<datasets.features._torchcodec.AudioDecoder object at 0x11642b6a0>
+```
+
+## AudioFolder
+
+You can also load a dataset with an `AudioFolder` dataset builder. It does not require writing a custom dataloader, making it useful for quickly creating and loading audio datasets with several thousand audio files.
+
+## AudioFolder with metadata
+
+To link your audio files with metadata information, make sure your dataset has a `metadata.csv` file. Your dataset structure might look like:
+
+```
+folder/train/metadata.csv
+folder/train/first_audio_file.mp3
+folder/train/second_audio_file.mp3
+folder/train/third_audio_file.mp3
+```
+
+Your `metadata.csv` file must have a `file_name` column which links audio files with their metadata. An example `metadata.csv` file might look like:
+
+```text
+file_name,transcription
+first_audio_file.mp3,znowu się duch z ciałem zrośnie w młodocianej wstaniesz wiosnie i możesz skutkiem tych leków umierać wstawać wiek wieków dalej tam były przestrogi jak siekać głowę jak nogi
+second_audio_file.mp3,już u źwierzyńca podwojów król zasiada przy nim książęta i panowie rada a gdzie wzniosły krążył ganek rycerze obok kochanek król skinął palcem zaczęto igrzysko
+third_audio_file.mp3,pewnie kędyś w obłędzie ubite minęły szlaki zaczekajmy dzień jaki poślemy szukać wszędzie dziś jutro pewnie będzie posłali wszędzie sługi czekali dzień i drugi gdy nic nie doczekali z płaczem chcą jechać dali
+```
+
+`AudioFolder` will load audio data and create a `transcription` column containing texts from `metadata.csv`:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_name")
+>>> # OR locally:
+>>> dataset = load_dataset("/path/to/folder")
+```
+
+For local datasets, this is equivalent to passing `audiofolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("audiofolder", data_dir="/path/to/folder")
+```
+
+Metadata can also be specified as JSON Lines, in which case use `metadata.jsonl` as the name of the metadata file. This format is helpful in scenarios when one of the columns is complex, e.g. a list of floats, to avoid parsing errors or reading the complex values as strings.
+
+To ignore the information in the metadata file, set `drop_metadata=True` in [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_with_metadata", drop_metadata=True)
+```
+
+If you don't have a metadata file, `AudioFolder` automatically infers the label name from the directory name.
+If you want to drop automatically created labels, set `drop_labels=True`.
+In this case, your dataset will only contain an audio column:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_without_metadata", drop_labels=True)
+```
+
+Finally the `filters` argument lets you load only a subset of the dataset, based on a condition on the label or the metadata. This is especially useful if the metadata is in Parquet format, since this format enables fast filtering. It is also recommended to use this argument with `streaming=True`, because by default the dataset is fully downloaded before filtering.
+
+```python
+>>> filters = [("label", "=", 0)]
+>>> dataset = load_dataset("username/dataset_name", streaming=True, filters=filters)
+```
+
+> [!TIP]
+> For more information about creating your own `AudioFolder` dataset, take a look at the [Create an audio dataset](./audio_dataset) guide.
+
+For a guide on how to load any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./loading">general loading guide</a>.
+
+## Audio decoding
+
+By default, audio files are decoded sequentially as torchcodec [`AudioDecoder`](https://docs.pytorch.org/torchcodec/stable/generated/torchcodec.decoders.AudioDecoder.html#torchcodec.decoders.AudioDecoder) objects when you iterate on a dataset.
+However it is possible to speed up the dataset significantly using multithreaded decoding:
+
+```python
+>>> import os
+>>> num_threads = num_threads = min(32, (os.cpu_count() or 1) + 4)
+>>> dataset = dataset.decode(num_threads=num_threads)
+>>> for example in dataset:  # up to 20 times faster !
+...     ...
+```
+
+You can enable multithreading using `num_threads`. This is especially useful to speed up remote data streaming.
+However it can be slower than `num_threads=0` for local data on fast disks.
+
+If you are not interested in the images decoded as NumPy arrays and would like to access the path/bytes instead, you can disable decoding:
+
+```python
+>>> dataset = dataset.decode(False)
+```
+
+Note: [`IterableDataset.decode`] is only available for streaming datasets at the moment.

docs/source/audio_process.mdx

@@ -0,0 +1,81 @@
+# Process audio data
+
+This guide shows specific methods for processing audio datasets. Learn how to:
+
+- Resample the sampling rate.
+- Use [`~Dataset.map`] with audio datasets.
+
+For a guide on how to process any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./process">general process guide</a>.
+
+## Cast
+
+The [`~Dataset.cast_column`] function is used to cast a column to another feature to be decoded. When you use this function with the [`Audio`] feature, you can resample the sampling rate:
+
+```py
+>>> from datasets import load_dataset, Audio
+
+>>> dataset = load_dataset("PolyAI/minds14", "en-US", split="train")
+>>> dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
+```
+
+Audio files are decoded and resampled on-the-fly, so the next time you access an example, the audio file is resampled to 16kHz:
+
+```py
+>>> audio = dataset[0]["audio"]
+<datasets.features._torchcodec.AudioDecoder object at 0x11642b6a0>
+>>> audio = audio_dataset[0]["audio"]
+>>> samples = audio.get_all_samples()
+>>> samples.data
+tensor([[ 0.0000e+00,  0.0000e+00,  0.0000e+00,  ...,  2.3447e-06,
+         -1.9127e-04, -5.3330e-05]]
+>>> samples.sample_rate
+16000
+```
+
+<div class="flex justify-center">
+  <img
+    class="block dark:hidden"
+    src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/resample.gif"
+  />
+  <img
+    class="hidden dark:block"
+    src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/resample-dark.gif"
+  />
+</div>
+
+## Map
+
+The [`~Dataset.map`] function helps preprocess your entire dataset at once. Depending on the type of model you're working with, you'll need to either load a [feature extractor](https://huggingface.co/docs/transformers/model_doc/auto#transformers.AutoFeatureExtractor) or a [processor](https://huggingface.co/docs/transformers/model_doc/auto#transformers.AutoProcessor).
+
+- For pretrained speech recognition models, load a feature extractor and tokenizer and combine them in a `processor`:
+
+  ```py
+  >>> from transformers import AutoTokenizer, AutoFeatureExtractor, AutoProcessor
+
+  >>> model_checkpoint = "facebook/wav2vec2-large-xlsr-53"
+  # after defining a vocab.json file you can instantiate a tokenizer object:
+  >>> tokenizer = AutoTokenizer("./vocab.json", unk_token="[UNK]", pad_token="[PAD]", word_delimiter_token="|")
+  >>> feature_extractor = AutoFeatureExtractor.from_pretrained(model_checkpoint)
+  >>> processor = AutoProcessor.from_pretrained(feature_extractor=feature_extractor, tokenizer=tokenizer)
+  ```
+
+- For fine-tuned speech recognition models, you only need to load a `processor`:
+
+  ```py
+  >>> from transformers import AutoProcessor
+
+  >>> processor = AutoProcessor.from_pretrained("facebook/wav2vec2-base-960h")
+  ```
+
+When you use [`~Dataset.map`] with your preprocessing function, include the `audio` column to ensure you're actually resampling the audio data:
+
+```py
+>>> def prepare_dataset(batch):
+...     audio = batch["audio"]
+...     batch["input_values"] = processor(audio.get_all_samples().data, sampling_rate=audio["sampling_rate"]).input_values[0]
+...     batch["input_length"] = len(batch["input_values"])
+...     with processor.as_target_processor():
+...         batch["labels"] = processor(batch["sentence"]).input_ids
+...     return batch
+>>> dataset = dataset.map(prepare_dataset, remove_columns=dataset.column_names)
+```

docs/source/cache.mdx

@@ -0,0 +1,110 @@
+# Cache management
+
+When you download a dataset from Hugging Face, the data are stored locally on your computer.
+Files from Hugging Face are stored as usual in the `huggingface_hub` cache, which is at `~/.cache/huggingface/hub` by default.
+See the [Hub cache documentation](https://huggingface.co/docs/huggingface_hub/guides/manage-cache) for more details and how to change its location.
+
+The Hub cache allows 🤗 Datasets to avoid re-downloading dataset files from Hugging Face every time you use them. 
+
+🤗 Datasets also has its own cache to store datasets converted in Arrow format (the format used by [`Dataset`] objects).
+
+This guide focuses on the 🤗 Datasets cache and will show you how to:
+
+- Change the cache directory.
+- Control how a dataset is loaded from the cache.
+- Clean up cache files in the directory.
+- Enable or disable caching.
+
+## Cache directory
+
+The default 🤗 Datasets cache directory is `~/.cache/huggingface/datasets`. Change the cache location by setting the shell environment variable, `HF_HOME` to another directory:
+
+```
+$ export HF_HOME="/path/to/another/directory/datasets"
+```
+
+Alternatively, you can set the `HF_DATASETS_CACHE` environment variable to control only the datasets-specific cache directory:
+
+```
+$ export HF_DATASETS_CACHE="/path/to/datasets_cache"
+```
+
+⚠️ This only applies to files written by the `datasets` library (e.g., Arrow files and indices).  
+It does **not** affect files downloaded from the Hugging Face Hub (such as models, tokenizers, or raw dataset sources), which are located in `~/.cache/huggingface/hub` by default and controlled separately via the `HF_HUB_CACHE` variable:
+
+```
+$ export HF_HUB_CACHE="/path/to/hub_cache"
+```
+
+💡 If you'd like to relocate all Hugging Face caches — including datasets and hub downloads — use the `HF_HOME` variable instead:
+
+```
+$ export HF_HOME="/path/to/cache_root"
+```
+
+This results in:
+- datasets cache → `/path/to/cache_root/datasets`
+- hub cache → `/path/to/cache_root/hub`
+
+These distinctions are especially useful when working in shared environments or networked file systems (e.g., NFS).  
+See [issue #7480](https://github.com/huggingface/datasets/issues/7480) for discussion on how users encountered unexpected cache locations when `HF_HUB_CACHE` was not set alongside `HF_DATASETS_CACHE`.
+
+When you load a dataset, you also have the option to change where the data is cached. Change the `cache_dir` parameter to the path you want:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('username/dataset', cache_dir="/path/to/another/directory/datasets")
+```
+
+## Download mode
+
+After you download a dataset, control how it is loaded by [`load_dataset`] with the `download_mode` parameter. By default, 🤗 Datasets will reuse a dataset if it exists. But if you need the original dataset without any processing functions applied, re-download the files as shown below:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('rajpurkar/squad', download_mode='force_redownload')
+```
+
+Refer to [`DownloadMode`] for a full list of download modes.
+
+## Cache files
+
+Clean up the Arrow cache files in the directory with [`Dataset.cleanup_cache_files`]:
+
+```py
+# Returns the number of removed cache files
+>>> dataset.cleanup_cache_files()
+2
+```
+
+## Enable or disable caching
+
+If you're using a cached file locally, it will automatically reload the dataset with any previous transforms you applied to the dataset. Disable this behavior by setting the argument `load_from_cache_file=False` in [`Dataset.map`]:
+
+```py
+>>> updated_dataset = small_dataset.map(add_prefix, load_from_cache_file=False)
+```
+
+In the example above, 🤗 Datasets will execute the function `add_prefix` over the entire dataset again instead of loading the dataset from its previous state.
+
+Disable caching on a global scale with [`disable_caching`]:
+
+```py
+>>> from datasets import disable_caching
+>>> disable_caching()
+```
+
+When you disable caching, 🤗 Datasets will no longer reload cached files when applying transforms to datasets. Any transform you apply on your dataset will be need to be reapplied.
+
+> [!TIP]
+> If you want to reuse a dataset from scratch, try setting the `download_mode` parameter in [`load_dataset`] instead.
+
+<a id='load_dataset_enhancing_performance'></a>
+
+## Improve performance
+
+Disabling the cache and copying the dataset in-memory will speed up dataset operations. There are two options for copying the dataset in-memory:
+
+1. Set `datasets.config.IN_MEMORY_MAX_SIZE` to a nonzero value (in bytes) that fits in your RAM memory.
+
+2. Set the environment variable `HF_DATASETS_IN_MEMORY_MAX_SIZE` to a nonzero value. Note that the first method takes higher precedence.

docs/source/cli.mdx

@@ -0,0 +1,48 @@
+# Command Line Interface (CLI)
+
+🤗 Datasets provides a command line interface (CLI) with useful shell commands to interact with your dataset.
+
+You can check the available commands:
+```bash
+>>> datasets-cli --help
+usage: datasets-cli <command> [<args>]
+
+positional arguments:
+  {env,test,delete_from_hub}
+                        datasets-cli command helpers
+    env                 Print relevant system environment info.
+    test                Test dataset loading.
+    delete_from_hub     Delete dataset config from the Hub
+
+optional arguments:
+  -h, --help            show this help message and exit
+```
+
+## Delete from Hub
+
+Delete a dataset configuration from a [supported dataset](repository_structure) on the Hub.
+
+```bash
+>>> datasets-cli delete_from_hub --help
+usage: datasets-cli <command> [<args>] delete_from_hub [-h] [--token TOKEN] [--revision REVISION] dataset_id config_name
+
+positional arguments:
+  dataset_id           source dataset ID, e.g. USERNAME/DATASET_NAME or ORGANIZATION/DATASET_NAME
+  config_name          config name to delete
+
+optional arguments:
+  -h, --help           show this help message and exit
+  --token TOKEN        access token to the Hugging Face Hub
+  --revision REVISION  source revision
+```
+
+For example:
+```bash
+>>> datasets-cli delete_from_hub USERNAME/DATASET_NAME CONFIG_NAME
+```
+
+> [!TIP]
+> Do not forget that you need to log in first to your Hugging Face account:
+> ```bash
+> >>> hf auth login
+> ```

docs/source/create_dataset.mdx

@@ -0,0 +1,125 @@
+# Create a dataset
+
+Sometimes, you may need to create a dataset if you're working with your own data. Creating a dataset with 🤗 Datasets confers all the advantages of the library to your dataset: fast loading and processing, [stream enormous datasets](stream), [memory-mapping](https://huggingface.co/course/chapter5/4?fw=pt#the-magic-of-memory-mapping), and more. You can easily and rapidly create a dataset with 🤗 Datasets low-code approaches, reducing the time it takes to start training a model. In many cases, it is as easy as [dragging and dropping](upload_dataset#upload-with-the-hub-ui) your data files into a dataset repository on the Hub.
+
+In this tutorial, you'll learn how to use 🤗 Datasets low-code methods for creating datasets from your own data.
+
+> **Note**
+>
+> If you want to learn how to load existing datasets, or how to load local and remote files (CSV, JSON, Parquet, etc.), see the [Load guide](https://huggingface.co/docs/datasets/loading).
+
+This tutorial covers:
+
+- Folder-based builders for quickly creating an image or audio dataset
+- `from_` methods for creating datasets from local Python objects
+
+## File-based builders
+
+🤗 Datasets supports many common formats such as `csv`, `json/jsonl`, `parquet`, `txt`.
+
+For example it can read a dataset made up of one or several CSV files (in this case, pass your CSV files as a list):
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("csv", data_files="my_file.csv")
+```
+
+To get the list of supported formats and code examples, follow this guide [here](https://huggingface.co/docs/datasets/loading#local-and-remote-files).
+
+## Folder-based builders
+
+There are two folder-based builders, [`ImageFolder`] and [`AudioFolder`]. These are low-code methods for quickly creating an image or speech and audio dataset with several thousand examples. They are great for rapidly prototyping computer vision and speech models before scaling to a larger dataset. Folder-based builders takes your data and automatically generates the dataset's features, splits, and labels. Under the hood:
+
+- [`ImageFolder`] uses the [`~datasets.Image`] feature to decode an image file. Many image extension formats are supported, such as jpg and png, but other formats are also supported. You can check the complete [list](https://github.com/huggingface/datasets/blob/b5672a956d5de864e6f5550e493527d962d6ae55/src/datasets/packaged_modules/imagefolder/imagefolder.py#L39) of supported image extensions.
+- [`AudioFolder`] uses the [`~datasets.Audio`] feature to decode an audio file. Extensions such as wav, mp3, and even mp4 are supported, and you can check the complete [list](https://ffmpeg.org/ffmpeg-formats.html) of supported audio extensions. Decoding is done via ffmpeg.
+
+The dataset splits are generated from the repository structure, and the label names are automatically inferred from the directory name.
+
+For example, if your image dataset (it is the same for an audio dataset) is stored like this:
+
+```
+pokemon/train/grass/bulbasaur.png
+pokemon/train/fire/charmander.png
+pokemon/train/water/squirtle.png
+
+pokemon/test/grass/ivysaur.png
+pokemon/test/fire/charmeleon.png
+pokemon/test/water/wartortle.png
+```
+
+Then this is how the folder-based builder generates an example:
+
+<div class="flex justify-center">
+  <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/folder-based-builder.png" />
+</div>
+
+Create the image dataset by specifying `imagefolder` in [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("imagefolder", data_dir="/path/to/pokemon")
+```
+
+An audio dataset is created in the same way, except you specify `audiofolder` in [`load_dataset`] instead:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("audiofolder", data_dir="/path/to/folder")
+```
+
+Any additional information about your dataset, such as text captions or transcriptions, can be included with a `metadata.csv` file in the folder containing your dataset. The metadata file needs to have a `file_name` column that links the image or audio file to its corresponding metadata:
+
+```
+file_name, text
+bulbasaur.png, There is a plant seed on its back right from the day this Pokémon is born.
+charmander.png, It has a preference for hot things.
+squirtle.png, When it retracts its long neck into its shell, it squirts out water with vigorous force.
+```
+
+To learn more about each of these folder-based builders, check out the <a href="https://huggingface.co/docs/datasets/image_dataset#imagefolder"><span class="underline decoration-yellow-400 decoration-2 font-semibold">ImageFolder</span></a>, <a href="https://huggingface.co/docs/datasets/audio_dataset#audiofolder"><span class="underline decoration-pink-400 decoration-2 font-semibold">AudioFolder</span></a>, or <a href="https://huggingface.co/docs/datasets/mesh_dataset#meshfolder"><span class="underline decoration-purple-400 decoration-2 font-semibold">MeshFolder</span></a> guides.
+
+## From Python dictionaries
+
+You can also create a dataset from data in Python dictionaries. There are two ways you can create a dataset using the `from_` methods:
+
+    * The [`~Dataset.from_generator`] method is the most memory-efficient way to create a dataset from a [generator](https://wiki.python.org/moin/Generators) due to a generators iterative behavior. This is especially useful when you're working with a really large dataset that may not fit in memory, since the dataset is generated on disk progressively and then memory-mapped.
+
+    ```py
+    >>> from datasets import Dataset
+    >>> def gen():
+    ...     yield {"pokemon": "bulbasaur", "type": "grass"}
+    ...     yield {"pokemon": "squirtle", "type": "water"}
+    >>> ds = Dataset.from_generator(gen)
+    >>> ds[0]
+    {"pokemon": "bulbasaur", "type": "grass"}
+    ```
+
+    A generator-based [`IterableDataset`] needs to be iterated over with a `for` loop for example:
+
+    ```py
+    >>> from datasets import IterableDataset
+    >>> ds = IterableDataset.from_generator(gen)
+    >>> for example in ds:
+    ...     print(example)
+    {"pokemon": "bulbasaur", "type": "grass"}
+    {"pokemon": "squirtle", "type": "water"}
+    ```
+
+    * The [`~Dataset.from_dict`] method is a straightforward way to create a dataset from a dictionary:
+
+    ```py
+    >>> from datasets import Dataset
+    >>> ds = Dataset.from_dict({"pokemon": ["bulbasaur", "squirtle"], "type": ["grass", "water"]})
+    >>> ds[0]
+    {"pokemon": "bulbasaur", "type": "grass"}
+    ```
+
+    To create an image or audio dataset, chain the [`~Dataset.cast_column`] method with [`~Dataset.from_dict`] and specify the column and feature type. For example, to create an audio dataset:
+
+    ```py
+    >>> audio_dataset = Dataset.from_dict({"audio": ["path/to/audio_1", ..., "path/to/audio_n"]}).cast_column("audio", Audio())
+    ```
+
+Now that you know how to create a dataset, consider sharing it on the Hub so the community can also benefit from your work! Go on to the next section to learn how to share your dataset.

docs/source/dataset_card.mdx

@@ -0,0 +1,27 @@
+# Create a dataset card
+
+Each dataset should have a dataset card to promote responsible usage and inform users of any potential biases within the dataset.
+This idea was inspired by the Model Cards proposed by [Mitchell, 2018](https://huggingface.co/papers/1810.03993).
+Dataset cards help users understand a dataset's contents, the context for using the dataset, how it was created, and any other considerations a user should be aware of.
+
+Creating a dataset card is easy and can be done in just a few steps:
+
+1. Go to your dataset repository on the [Hub](https://hf.co/new-dataset) and click on **Create Dataset Card** to create a new `README.md` file in your repository.
+
+2. Use the **Metadata UI** to select the tags that describe your dataset. You can add a license, language, pretty_name, the task_categories, size_categories, and any other tags that you think are relevant. These tags help users discover and find your dataset on the Hub.
+
+<div class="flex justify-center">
+    <img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-metadata-ui.png"/>
+    <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/hub/datasets-metadata-ui-dark.png"/>
+</div>
+
+  > [!TIP]
+  > For a complete, but not required, set of tag options you can also look at the [Dataset Card specifications](https://github.com/huggingface/hub-docs/blob/main/datasetcard.md?plain=1). This'll have a few more tag options like `multilinguality` and `language_creators` which are useful but not absolutely necessary.
+
+3. Click on the **Import dataset card template** link to automatically create a template with all the relevant fields to complete. Fill out the template sections to the best of your ability. Take a look at the [Dataset Card Creation Guide](https://github.com/huggingface/datasets/blob/main/templates/README_guide.md) for more detailed information about what to include in each section of the card. For fields you are unable to complete, you can write **[More Information Needed]**.
+
+4. Once you're done, commit the changes to the `README.md` file and you'll see the completed dataset card on your repository.
+
+YAML also allows you to customize the way your dataset is loaded by [defining splits and/or configurations](./repository_structure#define-your-splits-and-subsets-in-yaml) without the need to write any code.
+
+Feel free to take a look at the [SNLI](https://huggingface.co/datasets/stanfordnlp/snli), [CNN/DailyMail](https://huggingface.co/datasets/abisee/cnn_dailymail), and [Allociné](https://huggingface.co/datasets/tblard/allocine) dataset cards as examples to help you get started.

docs/source/depth_estimation.mdx

@@ -0,0 +1,231 @@
+# Depth estimation
+
+Depth estimation datasets are used to train a model to approximate the relative distance of every pixel in an
+image from the camera, also known as depth. The applications enabled by these datasets primarily lie in areas like visual machine
+perception and perception in robotics. Example applications include mapping streets for self-driving cars. This guide will show you how to apply transformations
+to a depth estimation dataset.
+
+Before you start, make sure you have up-to-date versions of `albumentations` installed:
+
+```bash
+pip install -U albumentations 
+```
+
+[Albumentations](https://albumentations.ai/) is a Python library for performing data augmentation
+for computer vision. It supports various computer vision tasks such as image classification, object
+detection, segmentation, and keypoint estimation.
+
+This guide uses the [NYU Depth V2](https://huggingface.co/datasets/sayakpaul/nyu_depth_v2) dataset which is 
+comprised of video sequences from various indoor scenes, recorded by RGB and depth cameras. The dataset consists of scenes from 3 cities and provides images along with
+their depth maps as labels.
+
+Load the `train` split of the dataset and take a look at an example:
+
+```py
+>>> from datasets import load_dataset
+
+>>> train_dataset = load_dataset("sayakpaul/nyu_depth_v2", split="train")
+>>> index = 17
+>>> example = train_dataset[index]
+>>> example
+{'image': <PIL.PngImagePlugin.PngImageFile image mode=RGB size=640x480>,
+ 'depth_map': <PIL.TiffImagePlugin.TiffImageFile image mode=F size=640x480>}
+```
+
+The dataset has two fields:
+
+* `image`: a PIL PNG image object with `uint8` data type.
+* `depth_map`: a PIL Tiff image object with `float32` data type which is the depth map of the image.
+
+Here the depth maps are using TIFF format as it supports a wide range of data types, including `float32` data.
+However it is mention-worthy that JPEG/PNG format can only store `uint8` or `uint16` data.
+Therefore if you have depth maps saved as JPEG/PNG, use the `Image(mode="F")` type to load them as single channel `float32` like normal depth maps:
+
+```python
+>>> from datasets import Image
+
+>>> train_dataset = train_dataset.cast_column("depth_map", Image(mode="F"))
+```
+
+Next, check out an image with:
+
+```py
+>>> example["image"]
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/depth_est_sample.png">
+</div>
+
+Before we look at the depth map, we need to first convert its data type to `uint8` using `.convert('RGB')` as PIL can't display `float32` images. Now take a look at its corresponding depth map:
+
+```py
+>>> example["depth_map"].convert("RGB")
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/depth_est_target.png">
+</div>
+
+It's all black! You'll need to add some color to the depth map to visualize it properly. To do that, either we can apply color automatically during display using `plt.imshow()` or create a colored depth map using `plt.cm` and then display it. In this example, we have used the latter one, as we can save/write the colored depth map later. (the utility below is taken from the [FastDepth repository](https://github.com/dwofk/fast-depth/blob/master/utils.py)).
+
+```py 
+>>> import numpy as np
+>>> import matplotlib.pyplot as plt
+
+>>> cmap = plt.cm.viridis
+
+>>> def colored_depthmap(depth, d_min=None, d_max=None):
+...     if d_min is None:
+...         d_min = np.min(depth)
+...     if d_max is None:
+...         d_max = np.max(depth)
+...     depth_relative = (depth - d_min) / (d_max - d_min)
+...     return 255 * cmap(depth_relative)[:,:,:3]
+
+>>> def show_depthmap(depth_map):
+...    if not isinstance(depth_map, np.ndarray):
+...        depth_map = np.array(depth_map)
+...    if depth_map.ndim == 3:
+...        depth_map = depth_map.squeeze()
+
+...    d_min = np.min(depth_map)
+...    d_max = np.max(depth_map)
+...    depth_map = colored_depthmap(depth_map, d_min, d_max)
+
+...    plt.imshow(depth_map.astype("uint8"))
+...    plt.axis("off")
+...    plt.show()
+
+>>> show_depthmap(example["depth_map"])
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/depth_est_target_viz.png">
+</div>
+
+You can also visualize several different images and their corresponding depth maps.
+
+```py
+>>> def merge_into_row(input_image, depth_target):
+...     if not isinstance(input_image, np.ndarray):
+...         input_image = np.array(input_image)
+...
+...     d_min = np.min(depth_target)
+...     d_max = np.max(depth_target)
+...     depth_target_col = colored_depthmap(depth_target, d_min, d_max)
+...     img_merge = np.hstack([input_image, depth_target_col])
+...
+...     return img_merge
+
+>>> random_indices = np.random.choice(len(train_dataset), 9).tolist()
+>>> plt.figure(figsize=(15, 6))
+>>> for i, idx in enumerate(random_indices):
+...     example = train_dataset[idx]
+...     ax = plt.subplot(3, 3, i + 1)
+...     image_viz = merge_into_row(
+...         example["image"], example["depth_map"]
+...     )
+...     plt.imshow(image_viz.astype("uint8"))
+...     plt.axis("off")
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/depth_est_collage.png">
+</div>
+
+Now apply some augmentations with `albumentations`. The augmentation transformations include:
+
+* Random horizontal flipping
+* Random cropping 
+* Random brightness and contrast 
+* Random gamma correction 
+* Random hue saturation
+
+```py 
+>>> import albumentations as A
+
+>>> crop_size = (448, 576)
+>>> transforms = [
+...     A.HorizontalFlip(p=0.5),
+...     A.RandomCrop(crop_size[0], crop_size[1]),
+...     A.RandomBrightnessContrast(),
+...     A.RandomGamma(),
+...     A.HueSaturationValue()
+... ]
+```
+
+Additionally, define a mapping to better reflect the target key name.
+
+```py 
+>>> additional_targets = {"depth": "mask"}
+>>> aug = A.Compose(transforms=transforms, additional_targets=additional_targets)
+```
+
+With `additional_targets` defined, you can pass the target depth maps to the `depth` argument of `aug` instead of `mask`. You'll notice this change
+in the `apply_transforms()` function defined below.
+
+Create a function to apply the transformation to the images as well as their depth maps:
+
+```py 
+>>> def apply_transforms(examples):
+...     transformed_images, transformed_maps = [], []
+...     for image, depth_map in zip(examples["image"], examples["depth_map"]):
+...         image, depth_map = np.array(image), np.array(depth_map)
+...         transformed = aug(image=image, depth=depth_map)
+...         transformed_images.append(transformed["image"])
+...         transformed_maps.append(transformed["depth"])
+...
+...     examples["pixel_values"] = transformed_images
+...     examples["labels"] = transformed_maps
+...     return examples
+```
+
+Use the [`~Dataset.set_transform`] function to apply the transformation on-the-fly to batches of the dataset to consume less disk space:
+
+```py
+>>> train_dataset.set_transform(apply_transforms)
+```
+
+You can verify the transformation worked by indexing into the `pixel_values` and `labels` of an example image:
+
+```py
+>>> example = train_dataset[index]
+
+>>> plt.imshow(example["pixel_values"])
+>>> plt.axis("off")
+>>> plt.show()
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/depth_est_sample_aug.png">
+</div>
+
+Visualize the same transformation on the image's corresponding depth map:
+
+```py 
+>>> show_depthmap(example["labels"])
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/depth_est_target_aug.png">
+</div>
+
+You can also visualize multiple training samples reusing the previous `random_indices`: 
+
+```py 
+>>> plt.figure(figsize=(15, 6))
+
+>>> for i, idx in enumerate(random_indices):
+...     ax = plt.subplot(3, 3, i + 1)
+...     example = train_dataset[idx]
+...     image_viz = merge_into_row(
+...         example["pixel_values"], example["labels"]
+...     )
+...     plt.imshow(image_viz.astype("uint8"))
+...     plt.axis("off")
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/depth_est_aug_collage.png">
+</div>

docs/source/document_dataset.mdx

@@ -0,0 +1,132 @@
+# Create a document dataset
+
+This guide will show you how to create a document dataset with `PdfFolder` and some metadata. This is a no-code solution for quickly creating a document dataset with several thousand PDFs.
+
+> [!TIP]
+> You can control access to your dataset by requiring users to share their contact information first. Check out the [Gated datasets](https://huggingface.co/docs/hub/datasets-gated) guide for more information about how to enable this feature on the Hub.
+
+## PdfFolder
+
+The `PdfFolder` is a dataset builder designed to quickly load a document dataset with several thousand PDFs without requiring you to write any code.
+
+> [!TIP]
+> 💡 Take a look at the [Split pattern hierarchy](repository_structure#split-pattern-hierarchy) to learn more about how `PdfFolder` creates dataset splits based on your dataset repository structure.
+
+`PdfFolder` automatically infers the class labels of your dataset based on the directory name. Store your dataset in a directory structure like:
+
+```
+folder/train/resume/0001.pdf
+folder/train/resume/0002.pdf
+folder/train/resume/0003.pdf
+
+folder/train/invoice/0001.pdf
+folder/train/invoice/0002.pdf
+folder/train/invoice/0003.pdf
+```
+
+If the dataset follows the `PdfFolder` structure, then you can load it directly with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("path/to/folder")
+```
+
+This is equivalent to passing `pdffolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("pdffolder", data_dir="/path/to/folder")
+```
+
+You can also use `pdffolder` to load datasets involving multiple splits. To do so, your dataset directory should have the following structure:
+
+```
+folder/train/resume/0001.pdf
+folder/train/resume/0002.pdf
+folder/test/invoice/0001.pdf
+folder/test/invoice/0002.pdf
+```
+
+> [!WARNING]
+> If all PDF files are contained in a single directory or if they are not on the same level of directory structure, `label` column won't be added automatically. If you need it, set `drop_labels=False` explicitly.
+
+
+If there is additional information you'd like to include about your dataset, like text captions or bounding boxes, add it as a `metadata.csv` file in your folder. This lets you quickly create datasets for different computer vision tasks like text captioning or object detection. You can also use a JSONL file `metadata.jsonl` or a Parquet file `metadata.parquet`.
+
+```
+folder/train/metadata.csv
+folder/train/0001.pdf
+folder/train/0002.pdf
+folder/train/0003.pdf
+```
+
+Your `metadata.csv` file must have a `file_name` or `*_file_name` field which links PDF files with their metadata:
+
+```csv
+file_name,additional_feature
+0001.pdf,This is a first value of a text feature you added to your pdfs
+0002.pdf,This is a second value of a text feature you added to your pdfs
+0003.pdf,This is a third value of a text feature you added to your pdfs
+```
+
+or using `metadata.jsonl`:
+
+```jsonl
+{"file_name": "0001.pdf", "additional_feature": "This is a first value of a text feature you added to your PDFs"}
+{"file_name": "0002.pdf", "additional_feature": "This is a second value of a text feature you added to your PDFs"}
+{"file_name": "0003.pdf", "additional_feature": "This is a third value of a text feature you added to your PDFs"}
+```
+
+Here the `file_name` must be the name of the PDF file next to the metadata file. More generally, it must be the relative path from the directory containing the metadata to the PDF file.
+
+It's possible to point to more than one PDF in each row in your dataset, for example if both your input and output are pdfs:
+
+```jsonl
+{"input_file_name": "0001.pdf", "output_file_name": "0001_output.pdf"}
+{"input_file_name": "0002.pdf", "output_file_name": "0002_output.pdf"}
+{"input_file_name": "0003.pdf", "output_file_name": "0003_output.pdf"}
+```
+
+You can also define lists of PDFs. In that case you need to name the field `file_names` or `*_file_names`. Here is an example:
+
+```jsonl
+{"pdfs_file_names": ["0001_part1.pdf", "0001_part2.pdf"], "label": "urgent"}
+{"pdfs_file_names": ["0002_part1.pdf", "0002_part2.pdf"], "label": "urgent"}
+{"pdfs_file_names": ["0003_part1.pdf", "0002_part2.pdf"], "label": "normal"}
+```
+
+### OCR (Optical Character Recognition)
+
+OCR datasets have the text contained in a PDF. An example `metadata.csv` may look like:
+
+```csv
+file_name,text
+0001.pdf,Invoice 1234 from 01/01/1970...
+0002.pdf,Software Engineer Resume. Education: ...
+0003.pdf,Attention is all you need. Abstract. The ...
+```
+
+Load the dataset with `PdfFolder`, and it will create a `text` column for the PDF captions:
+
+```py
+>>> dataset = load_dataset("pdffolder", data_dir="/path/to/folder", split="train")
+>>> dataset[0]["text"]
+"Invoice 1234 from 01/01/1970..."
+```
+
+### Upload dataset to the Hub
+
+Once you've created a dataset, you can share it to the using `huggingface_hub` for example. Make sure you have the [huggingface_hub](https://huggingface.co/docs/huggingface_hub/index) library installed and you're logged in to your Hugging Face account (see the [Upload with Python tutorial](upload_dataset#upload-with-python) for more details).
+
+Upload your dataset with `huggingface_hub.HfApi.upload_folder`:
+
+```py
+from huggingface_hub import HfApi
+api = HfApi()
+
+api.upload_folder(
+    folder_path="/path/to/local/dataset",
+    repo_id="username/my-cool-dataset",
+    repo_type="dataset",
+)
+```

docs/source/document_load.mdx

@@ -0,0 +1,204 @@
+# Load pdf data
+
+> [!WARNING]
+> Pdf support is experimental and is subject to change.
+
+Pdf datasets have [`Pdf`] type columns, which contain `pdfplumber` objects. 
+
+> [!TIP]
+> To work with pdf datasets, you need to have the `pdfplumber` package installed. Check out the [installation](https://github.com/jsvine/pdfplumber#installation) guide to learn how to install it.
+
+When you load a pdf dataset and call the pdf column, the pdfs are decoded as `pdfplumber` Pdfs:
+
+```py
+>>> from datasets import load_dataset, Pdf
+
+>>> dataset = load_dataset("path/to/pdf/folder", split="train")
+>>> dataset[0]["pdf"]
+<pdfplumber.pdf.PDF at 0x1075bc320>
+```
+
+> [!WARNING]
+> Index into a pdf dataset using the row index first and then the `pdf` column - `dataset[0]["pdf"]` - to avoid creating all the pdf objects in the dataset. Otherwise, this can be a slow and time-consuming process if you have a large dataset.
+
+For a guide on how to load any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./loading">general loading guide</a>.
+
+## Read pages
+
+Access pages directly from a pdf using the `.pages` attribute.
+
+Then you can use the `pdfplumber` functions to read texts, tables and images, e.g.:
+
+```python
+>>> pdf = dataset[0]["pdf"]
+>>> first_page = pdf.pages[0]
+>>> first_page
+<Page:1>
+>>> first_page.extract_text()
+Docling Technical Report
+Version1.0
+ChristophAuer MaksymLysak AhmedNassar MicheleDolfi NikolaosLivathinos
+PanosVagenas CesarBerrospiRamis MatteoOmenetti FabianLindlbauer
+KasperDinkla LokeshMishra YusikKim ShubhamGupta RafaelTeixeiradeLima
+ValeryWeber LucasMorin IngmarMeijer ViktorKuropiatnyk PeterW.J.Staar
+AI4KGroup,IBMResearch
+Ru¨schlikon,Switzerland
+Abstract
+This technical report introduces Docling, an easy to use, self-contained, MIT-
+licensed open-source package for PDF document conversion.
+...
+>>> first_page.images
+In [24]: first_page.images
+Out[24]: 
+[{'x0': 256.5,
+  'y0': 621.0,
+  'x1': 355.49519999999995,
+  'y1': 719.9952,
+  'width': 98.99519999999995,
+  'height': 98.99519999999995,
+  'name': 'Im1',
+  'stream': <PDFStream(44): raw=88980, {'Type': /'XObject', 'Subtype': /'Image', 'BitsPerComponent': 8, 'ColorSpace': /'DeviceRGB', 'Filter': /'DCTDecode', 'Height': 1024, 'Length': 88980, 'Width': 1024}>,
+  'srcsize': (1024, 1024),
+  'imagemask': None,
+  'bits': 8,
+  'colorspace': [/'DeviceRGB'],
+  'mcid': None,
+  'tag': None,
+  'object_type': 'image',
+  'page_number': 1,
+  'top': 72.00480000000005,
+  'bottom': 171.0,
+  'doctop': 72.00480000000005}]
+>>> first_page.extract_tables()
+[]
+```
+
+You can also load each page as a `PIL.Image`:
+
+```python
+>>> import PIL.Image
+>>> import io
+>>> first_page.to_image()
+<pdfplumber.display.PageImage at 0x107d68dd0>
+>>> buffer = io.BytesIO()
+>>> first_page.to_image().save(buffer)
+>>> img = PIL.Image.open(buffer)
+>>> img
+<PIL.PngImagePlugin.PngImageFile image mode=P size=612x792>
+```
+
+Note that you can pass `resolution=` to `.to_image()` to render the image in higher resolution that the default (72 ppi).
+
+## Local files
+
+You can load a dataset from the pdf path. Use the [`~Dataset.cast_column`] function to accept a column of pdf file paths, and decode it into a `pdfplumber` pdf with the [`Pdf`] feature:
+```py
+>>> from datasets import Dataset, Pdf
+
+>>> dataset = Dataset.from_dict({"pdf": ["path/to/pdf_1", "path/to/pdf_2", ..., "path/to/pdf_n"]}).cast_column("pdf", Pdf())
+>>> dataset[0]["pdf"]
+<pdfplumber.pdf.PDF at 0x1657d0280>
+```
+
+If you only want to load the underlying path to the pdf dataset without decoding the pdf object, set `decode=False` in the [`Pdf`] feature:
+
+```py
+>>> dataset = dataset.cast_column("pdf", Pdf(decode=False))
+>>> dataset[0]["pdf"]
+{'bytes': None,
+ 'path': 'path/to/pdf/folder/pdf0.pdf'}
+```
+
+## PdfFolder
+
+You can also load a dataset with an `PdfFolder` dataset builder which does not require writing a custom dataloader. This makes `PdfFolder` ideal for quickly creating and loading pdf datasets with several thousand pdfs for different vision tasks. Your pdf dataset structure should look like this:
+
+```
+folder/train/resume/0001.pdf
+folder/train/resume/0002.pdf
+folder/train/resume/0003.pdf
+
+folder/train/invoice/0001.pdf
+folder/train/invoice/0002.pdf
+folder/train/invoice/0003.pdf
+```
+
+If the dataset follows the `PdfFolder` structure, then you can load it directly with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_name")
+>>> # OR locally:
+>>> dataset = load_dataset("/path/to/folder")
+```
+
+For local datasets, this is equivalent to passing `pdffolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("pdffolder", data_dir="/path/to/folder")
+```
+
+Then you can access the pdfs as `pdfplumber.pdf.PDF` objects:
+
+```
+>>> dataset["train"][0]
+{"pdf": <pdfplumber.pdf.PDF at 0x161715e50>, "label": 0}
+
+>>> dataset["train"][-1]
+{"pdf": <pdfplumber.pdf.PDF at 0x16170bd90>, "label": 1}
+```
+
+To ignore the information in the metadata file, set `drop_metadata=True` in [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_with_metadata", drop_metadata=True)
+```
+
+If you don't have a metadata file, `PdfFolder` automatically infers the label name from the directory name.
+If you want to drop automatically created labels, set `drop_labels=True`.
+In this case, your dataset will only contain a pdf column:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_without_metadata", drop_labels=True)
+```
+
+Finally the `filters` argument lets you load only a subset of the dataset, based on a condition on the label or the metadata. This is especially useful if the metadata is in Parquet format, since this format enables fast filtering. It is also recommended to use this argument with `streaming=True`, because by default the dataset is fully downloaded before filtering.
+
+```python
+>>> filters = [("label", "=", 0)]
+>>> dataset = load_dataset("username/dataset_name", streaming=True, filters=filters)
+```
+
+> [!TIP]
+> For more information about creating your own `PdfFolder` dataset, take a look at the [Create a pdf dataset](./document_dataset) guide.
+
+## Pdf decoding
+
+By default, pdfs are decoded sequentially as pdfplumber `PDFs` when you iterate on a dataset.
+It sequentially decodes the metadata of the pdfs, and doesn't read the pdf pages until you access them.
+
+However it is possible to speed up the dataset significantly using multithreaded decoding:
+
+```python
+>>> import os
+>>> num_threads = num_threads = min(32, (os.cpu_count() or 1) + 4)
+>>> dataset = dataset.decode(num_threads=num_threads)
+>>> for example in dataset:  # up to 20 times faster !
+...     ...
+```
+
+You can enable multithreading using `num_threads`. This is especially useful to speed up remote data streaming.
+However it can be slower than `num_threads=0` for local data on fast disks.
+
+If you are not interested in the documents decoded as pdfplumber `PDFs` and would like to access the path/bytes instead, you can disable decoding:
+
+```python
+>>> dataset = dataset.decode(False)
+```
+
+Note: [`IterableDataset.decode`] is only available for streaming datasets at the moment.

docs/source/faiss_es.mdx

@@ -0,0 +1,133 @@
+# Search index
+
+[FAISS](https://github.com/facebookresearch/faiss) and [Elasticsearch](https://www.elastic.co/elasticsearch/) enables searching for examples in a dataset. This can be useful when you want to retrieve specific examples from a dataset that are relevant to your NLP task. For example, if you are working on an Open Domain Question Answering task, you may want to only return examples that are relevant to answering your question.
+
+This guide will show you how to build an index for your dataset that will allow you to search it.
+
+## FAISS
+
+FAISS retrieves documents based on the similarity of their vector representations. In this example, you will generate the vector representations with the [DPR](https://huggingface.co/transformers/model_doc/dpr.html) model.
+
+1. Download the DPR model from 🤗 Transformers:
+
+```py
+>>> from transformers import DPRContextEncoder, DPRContextEncoderTokenizer
+>>> import torch
+>>> torch.set_grad_enabled(False)
+>>> ctx_encoder = DPRContextEncoder.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
+>>> ctx_tokenizer = DPRContextEncoderTokenizer.from_pretrained("facebook/dpr-ctx_encoder-single-nq-base")
+```
+
+2. Load your dataset and compute the vector representations:
+
+```py
+>>> from datasets import load_dataset
+>>> ds = load_dataset('community-datasets/crime_and_punish', split='train[:100]')
+>>> ds_with_embeddings = ds.map(lambda example: {'embeddings': ctx_encoder(**ctx_tokenizer(example["line"], return_tensors="pt"))[0][0].numpy()})
+```
+
+3. Create the index with [`Dataset.add_faiss_index`]:
+
+```py
+>>> ds_with_embeddings.add_faiss_index(column='embeddings')
+```
+
+4. Now you can query your dataset with the `embeddings` index. Load the DPR Question Encoder, and search for a question with [`Dataset.get_nearest_examples`]:
+
+```py
+>>> from transformers import DPRQuestionEncoder, DPRQuestionEncoderTokenizer
+>>> q_encoder = DPRQuestionEncoder.from_pretrained("facebook/dpr-question_encoder-single-nq-base")
+>>> q_tokenizer = DPRQuestionEncoderTokenizer.from_pretrained("facebook/dpr-question_encoder-single-nq-base")
+
+>>> question = "Is it serious ?"
+>>> question_embedding = q_encoder(**q_tokenizer(question, return_tensors="pt"))[0][0].numpy()
+>>> scores, retrieved_examples = ds_with_embeddings.get_nearest_examples('embeddings', question_embedding, k=10)
+>>> retrieved_examples["line"][0]
+'_that_ serious? It is not serious at all. It’s simply a fantasy to amuse\r\n'
+```
+
+5. You can access the index with [`Dataset.get_index`] and use it for special operations, e.g. query it using `range_search`:
+
+```py
+>>> faiss_index = ds_with_embeddings.get_index('embeddings').faiss_index
+>>> limits, distances, indices = faiss_index.range_search(x=question_embedding.reshape(1, -1), thresh=0.95)
+```
+
+6. When you are done querying, save the index on disk with [`Dataset.save_faiss_index`]:
+
+```py
+>>> ds_with_embeddings.save_faiss_index('embeddings', 'my_index.faiss')
+```
+
+7. Reload it at a later time with [`Dataset.load_faiss_index`]:
+
+```py
+>>> ds = load_dataset('community-datasets/crime_and_punish', split='train[:100]')
+>>> ds.load_faiss_index('embeddings', 'my_index.faiss')
+```
+
+## Elasticsearch
+
+Unlike FAISS, Elasticsearch retrieves documents based on exact matches. 
+
+Start Elasticsearch on your machine, or see the [Elasticsearch installation guide](https://www.elastic.co/guide/en/elasticsearch/reference/current/setup.html) if you don't already have it installed.
+
+1. Load the dataset you want to index:
+
+```py
+>>> from datasets import load_dataset
+>>> squad = load_dataset('rajpurkar/squad', split='validation')
+```
+
+2. Build the index with [`Dataset.add_elasticsearch_index`]:
+
+```py
+>>> squad.add_elasticsearch_index("context", host="localhost", port="9200")
+```
+
+3. Then you can query the `context` index with [`Dataset.get_nearest_examples`]:
+
+```py
+>>> query = "machine"
+>>> scores, retrieved_examples = squad.get_nearest_examples("context", query, k=10)
+>>> retrieved_examples["title"][0]
+'Computational_complexity_theory'
+```
+
+4. If you want to reuse the index, define the `es_index_name` parameter when you build the index:
+
+```py
+>>> from datasets import load_dataset
+>>> squad = load_dataset('rajpurkar/squad', split='validation')
+>>> squad.add_elasticsearch_index("context", host="localhost", port="9200", es_index_name="hf_squad_val_context")
+>>> squad.get_index("context").es_index_name
+hf_squad_val_context
+```
+
+5. Reload it later with the index name when you call [`Dataset.load_elasticsearch_index`]:
+
+```py
+>>> from datasets import load_dataset
+>>> squad = load_dataset('rajpurkar/squad', split='validation')
+>>> squad.load_elasticsearch_index("context", host="localhost", port="9200", es_index_name="hf_squad_val_context")
+>>> query = "machine"
+>>> scores, retrieved_examples = squad.get_nearest_examples("context", query, k=10)
+```
+
+For more advanced Elasticsearch usage, you can specify your own configuration with custom settings:
+
+```py
+>>> import elasticsearch as es
+>>> import elasticsearch.helpers
+>>> from elasticsearch import Elasticsearch
+>>> es_client = Elasticsearch([{"host": "localhost", "port": "9200"}])  # default client
+>>> es_config = {
+...     "settings": {
+...         "number_of_shards": 1,
+...         "analysis": {"analyzer": {"stop_standard": {"type": "standard", " stopwords": "_english_"}}},
+...     },
+...     "mappings": {"properties": {"text": {"type": "text", "analyzer": "standard", "similarity": "BM25"}}},
+... }  # default config
+>>> es_index_name = "hf_squad_context"  # name of the index in Elasticsearch
+>>> squad.add_elasticsearch_index("context", es_client=es_client, es_config=es_config, es_index_name=es_index_name)
+```

docs/source/filesystems.mdx

@@ -0,0 +1,124 @@
+# Cloud storage
+
+## Hugging Face Datasets
+
+The Hugging Face Dataset Hub is home to a growing collection of datasets that span a variety of domains and tasks.
+
+It's more than a cloud storage: the Dataset Hub is a platform that provides data versioning thanks to git, as well as a Dataset Viewer to explore the data, making it a great place to store AI-ready datasets.
+
+This guide shows how to import data from other cloud storage using the filesystems implementations from `fsspec`.
+
+## Hugging Face Storage Buckets
+
+Storage Buckets are a repo type on the Hugging Face Hub providing S3-like object storage, powered by the Xet storage backend. Unlike Git-based dataset repositories, buckets are non-versioned and mutable, designed for use cases where you need simple, fast storage such as logs, intermediate artifacts, or any large collection of files that doesn’t need version control.
+
+## Import data from a cloud storage
+
+Most cloud storage providers have a `fsspec` FileSystem implementation, which is useful to import data from any cloud provider with the same code.
+This is especially useful to publish datasets on Hugging Face.
+
+Take a look at the following table for some example of supported cloud storage providers:
+
+| Storage provider     | Filesystem implementation                                     |
+|----------------------|---------------------------------------------------------------|
+| Amazon S3            | [s3fs](https://s3fs.readthedocs.io/en/latest/)                |
+| Google Cloud Storage | [gcsfs](https://gcsfs.readthedocs.io/en/latest/)              |
+| Azure Blob/DataLake  | [adlfs](https://github.com/fsspec/adlfs)                      |
+| Oracle Cloud Storage | [ocifs](https://ocifs.readthedocs.io/en/latest/)              |
+
+This guide will show you how to import data files from any cloud storage and save a dataset on Hugging Face.
+
+Let's say we want to publish a dataset on Hugging Face from Parquet files from a cloud storage.
+
+First, instantiate your cloud storage filesystem and list the files you'd like to import:
+
+```python
+>>> import fsspec
+>>> fs = fsspec.filesystem("...")  # s3 / gcs / abfs / adl / oci / ...
+>>> data_dir = "path/to/my/data/"
+>>> pattern = "*.parquet"
+>>> data_files = fs.glob(data_dir + pattern)
+["path/to/my/data/0001.parquet", "path/to/my/data/0001.parquet", ...]
+```
+
+### Publish a Dataset
+
+Then you can create a dataset on Hugging Face and import the data files, using for example:
+
+```python
+>>> from huggingface_hub import create_repo, upload_folder
+>>> from tqdm.auto import tqdm
+>>> destination_dataset = "username/my-dataset"
+>>> create_repo(destination_dataset, repo_type="dataset")
+>>> batch_size = 100
+>>> for data_files in batched(tqdm(fs.glob(data_dir + pattern)), batch_size):
+...     with TemporaryDirectory() as tmp_dir:
+...         tmp_files = [os.path.join(tmp_dir, x[len(data_dir):]) for x in data_files]
+...         fs.download(data_files, tmp_files)
+...         upload_folder(
+...             repo_id=destination_dataset,
+...             folder_path=tmp_dir,
+...             repo_type="dataset",
+...         )
+```
+
+Check out the [huggingface_hub](https://huggingface.co/docs/huggingface_hub) documentation on files uploads [here](https://huggingface.co/docs/huggingface_hub/en/guides/upload) if you're looking for more upload options.
+
+Finally you can now load the dataset using 🤗 Datasets:
+
+```python
+>>> from datasets import load_dataset
+>>> ds = load_dataset("username/my-dataset")
+```
+
+### Import raw data to Storage Buckets
+
+Alternatively if you wish not to publish a dataset but simply import raw data files in a Hugging Face [Storage Bucket](https://huggingface.co/docs/hub/storage-buckets), you can use:
+
+```python
+>>> from huggingface_hub import create_bucket, sync_bucket
+>>> from tqdm.auto import tqdm
+>>> from itertools import batched
+>>> from tempfile import TemporaryDirectory
+>>> import os
+>>> create_bucket("username/my-bucket")
+>>> bucket_files_location = "hf://buckets/username/my-bucket/path/to/raw/files"
+>>> batch_size = 100
+>>> for data_files in batched(tqdm(fs.glob(data_dir + pattern)), batch_size):
+...     with TemporaryDirectory() as tmp_dir:
+...         tmp_files = [os.path.join(tmp_dir, x[len(data_dir):]) for x in data_files]
+...         fs.download(data_files, tmp_files)
+...         sync_bucket(tmp_dir, bucket_files_location)
+```
+
+Check out the [huggingface_hub](https://huggingface.co/docs/huggingface_hub) documentation on Storage Buckets [here](https://huggingface.co/docs/hub/storage-buckets) if you're looking for more upload options.
+
+Then later you can load the raw files using 🤗 Datasets, transform them and upload the final AI-ready datasets, e.g. in a streaming manner:
+
+If the files are in a format supported by 🤗 Datasets:
+
+```python
+>>> from datasets import load_dataset
+>>> ds = load_dataset(bucket_files_location, streaming=True)
+>>> ds = ds.map(...).filter(...)
+>>> ds.push_to_hub("username/my-dataset", num_proc=4)
+>>> # and later
+>>> ds = load_dataset("username/my-dataset")
+```
+
+Otherwise you can use your own file parsing function:
+
+```python
+>>> from datasets import IterableDataset
+>>> from huggingface_hub import hffs
+>>> data_files = hffs.find(bucket_files_location)
+>>> num_shards = 1024  # For parallelism. PS: every shard should fit in RAM
+>>> ds = IterableDataset.from_dict({"data_file": data_files}, num_shards=num_shards)
+>>> def parse_data_files(data_files):
+...     ...
+...     return {"col_1": [...], "col_2": [...]}
+>>> ds = ds.map(parse_data_files, batched=True, input_column=["data_file"])
+>>> ds.push_to_hub("username/my-dataset", num_proc=4)
+>>> # and later
+>>> ds = load_dataset("username/my-dataset")
+```

docs/source/how_to.md

@@ -0,0 +1,19 @@
+# Overview
+
+The how-to guides offer a more comprehensive overview of all the tools 🤗 Datasets offers and how to use them. This will help you tackle messier real-world datasets where you may need to manipulate the dataset structure or content to get it ready for training.
+
+The guides assume you are familiar and comfortable with the 🤗 Datasets basics. We recommend newer users check out our [tutorials](tutorial) first.
+
+> [!TIP]
+> Interested in learning more? Take a look at [Chapter 5](https://huggingface.co/course/chapter5/1?fw=pt) of the Hugging Face course!
+
+The guides are organized into six sections:
+
+- <span class="underline decoration-sky-400 decoration-2 font-semibold">General usage</span>: Functions for general dataset loading and processing. The functions shown in this section are applicable across all dataset modalities.
+- <span class="underline decoration-pink-400 decoration-2 font-semibold">Audio</span>: How to load, process, and share audio datasets.
+- <span class="underline decoration-yellow-400 decoration-2 font-semibold">Vision</span>: How to load, process, and share image and video datasets.
+- <span class="underline decoration-green-400 decoration-2 font-semibold">Text</span>: How to load, process, and share text datasets.
+- <span class="underline decoration-orange-400 decoration-2 font-semibold">Tabular</span>: How to load, process, and share tabular datasets.
+- <span class="underline decoration-indigo-400 decoration-2 font-semibold">Dataset repository</span>: How to share and upload a dataset to the <a href="https://huggingface.co/datasets">Hub</a>.
+
+If you have any questions about 🤗 Datasets, feel free to join and ask the community on our [forum](https://discuss.huggingface.co/c/datasets/10).

docs/source/image_classification.mdx

@@ -0,0 +1,86 @@
+# Image classification
+
+Image classification datasets are used to train a model to classify an entire image. There are a wide variety of applications enabled by these datasets such as identifying endangered wildlife species or screening for disease in medical images. This guide will show you how to apply transformations to an image classification dataset.
+
+Before you start, make sure you have up-to-date versions of `albumentations` and `cv2` installed:
+
+```bash
+pip install -U albumentations opencv-python
+```
+
+This guide uses the [Beans](https://huggingface.co/datasets/beans) dataset for identifying the type of bean plant disease based on an image of its leaf.
+
+Load the dataset and take a look at an example:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("AI-Lab-Makerere/beans")
+>>> dataset["train"][10]
+{'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=500x500 at 0x7F8D2F4D7A10>,
+ 'image_file_path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/angular_leaf_spot/angular_leaf_spot_train.204.jpg',
+ 'labels': 0}
+```
+
+The dataset has three fields:
+
+* `image`: a PIL image object.
+* `image_file_path`: the path to the image file.
+* `labels`: the label or category of the image.
+
+Next, check out an image:
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/img_clf.png">
+</div>
+
+Now apply some augmentations with `albumentations`. You'll randomly crop the image, flip it horizontally, and adjust its brightness.
+
+```py
+>>> import cv2
+>>> import albumentations
+>>> import numpy as np
+
+>>> transform = albumentations.Compose([
+...     albumentations.RandomCrop(width=256, height=256),
+...     albumentations.HorizontalFlip(p=0.5),
+...     albumentations.RandomBrightnessContrast(p=0.2),
+... ])
+```
+
+Create a function to apply the transformation to the images:
+
+```py
+>>> def transforms(examples):
+...     examples["pixel_values"] = [
+...         transform(image=np.array(image))["image"] for image in examples["image"]
+...     ]
+... 
+...     return examples
+```
+
+Use the [`~Dataset.set_transform`] function to apply the transformation on-the-fly to batches of the dataset to consume less disk space:
+
+```py
+>>> dataset.set_transform(transforms)
+```
+
+You can verify the transformation worked by indexing into the `pixel_values` of the first example:
+
+```py
+>>> import numpy as np
+>>> import matplotlib.pyplot as plt
+
+>>> img = dataset["train"][0]["pixel_values"]
+>>> plt.imshow(img)
+```
+
+<div class="flex justify-center">
+    <img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/img_clf_aug.png">
+    <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/img_clf_aug.png"/>
+</div>
+
+> [!TIP]
+> Now that you know how to process a dataset for image classification, learn
+> [how to train an image classification model](https://colab.research.google.com/github/huggingface/notebooks/blob/main/examples/image_classification.ipynb)
+> and use it for inference.

docs/source/image_dataset.mdx

@@ -0,0 +1,297 @@
+# Create an image dataset
+
+There are two methods for creating and sharing an image dataset. This guide will show you how to:
+
+* Create an image dataset from local files in python with [`Dataset.push_to_hub`]. This is an easy way that requires only a few steps in python.
+
+* Create an image dataset with `ImageFolder` and some metadata. This is a no-code solution for quickly creating an image dataset with several thousand images.
+
+> [!TIP]
+> You can control access to your dataset by requiring users to share their contact information first. Check out the [Gated datasets](https://huggingface.co/docs/hub/datasets-gated) guide for more information about how to enable this feature on the Hub.
+
+## ImageFolder
+
+The `ImageFolder` is a dataset builder designed to quickly load an image dataset with several thousand images without requiring you to write any code.
+
+> [!TIP]
+> 💡 Take a look at the [Split pattern hierarchy](repository_structure#split-pattern-hierarchy) to learn more about how `ImageFolder` creates dataset splits based on your dataset repository structure.
+
+`ImageFolder` automatically infers the class labels of your dataset based on the directory name. Store your dataset in a directory structure like:
+
+```
+folder/train/dog/golden_retriever.png
+folder/train/dog/german_shepherd.png
+folder/train/dog/chihuahua.png
+
+folder/train/cat/maine_coon.png
+folder/train/cat/bengal.png
+folder/train/cat/birman.png
+```
+
+If the dataset follows the `ImageFolder` structure, then you can load it directly with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("path/to/folder")
+```
+
+This is equivalent to passing `imagefolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder")
+```
+
+You can also use `imagefolder` to load datasets involving multiple splits. To do so, your dataset directory should have the following structure:
+
+```
+folder/train/dog/golden_retriever.png
+folder/train/cat/maine_coon.png
+folder/test/dog/german_shepherd.png
+folder/test/cat/bengal.png
+```
+
+> [!WARNING]
+> If all image files are contained in a single directory or if they are not on the same level of directory structure, `label` column won't be added automatically. If you need it, set `drop_labels=False` explicitly.
+
+
+If there is additional information you'd like to include about your dataset, like text captions or bounding boxes, add it as a `metadata.csv` file in your folder. This lets you quickly create datasets for different computer vision tasks like text captioning or object detection. You can also use a JSONL file `metadata.jsonl` or a Parquet file `metadata.parquet`.
+
+```
+folder/train/metadata.csv
+folder/train/0001.png
+folder/train/0002.png
+folder/train/0003.png
+```
+
+You can also zip your images, and in this case each zip should contain both the images and the metadata
+
+```
+folder/train.zip
+folder/test.zip
+folder/validation.zip
+```
+
+Your `metadata.csv` file must have a `file_name` or `*_file_name` field which links image files with their metadata:
+
+```csv
+file_name,additional_feature
+0001.png,This is a first value of a text feature you added to your images
+0002.png,This is a second value of a text feature you added to your images
+0003.png,This is a third value of a text feature you added to your images
+```
+
+or using `metadata.jsonl`:
+
+```jsonl
+{"file_name": "0001.png", "additional_feature": "This is a first value of a text feature you added to your images"}
+{"file_name": "0002.png", "additional_feature": "This is a second value of a text feature you added to your images"}
+{"file_name": "0003.png", "additional_feature": "This is a third value of a text feature you added to your images"}
+```
+
+Here the `file_name` must be the name of the image file next to the metadata file. More generally, it must be the relative path from the directory containing the metadata to the image file.
+
+It's possible to point to more than one image in each row in your dataset, for example if both your input and output are images:
+
+```jsonl
+{"input_file_name": "0001.png", "output_file_name": "0001_output.png"}
+{"input_file_name": "0002.png", "output_file_name": "0002_output.png"}
+{"input_file_name": "0003.png", "output_file_name": "0003_output.png"}
+```
+
+You can also define lists of images. In that case you need to name the field `file_names` or `*_file_names`. Here is an example:
+
+```jsonl
+{"frames_file_names": ["0001_t0.png", "0001_t1.png"], label: "moving_up"}
+{"frames_file_names": ["0002_t0.png", "0002_t1.png"], label: "moving_down"}
+{"frames_file_names": ["0003_t0.png", "0003_t1.png"], label: "moving_right"}
+```
+
+### Image captioning
+
+Image captioning datasets have text describing an image. An example `metadata.csv` may look like:
+
+```csv
+file_name,text
+0001.png,This is a golden retriever playing with a ball
+0002.png,A german shepherd
+0003.png,One chihuahua
+```
+
+Load the dataset with `ImageFolder`, and it will create a `text` column for the image captions:
+
+```py
+>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder", split="train")
+>>> dataset[0]["text"]
+"This is a golden retriever playing with a ball"
+```
+
+### Object detection
+
+Object detection datasets have bounding boxes and categories identifying objects in an image. An example `metadata.jsonl` may look like:
+
+```jsonl
+{"file_name": "0001.png", "objects": {"bbox": [[302.0, 109.0, 73.0, 52.0]], "categories": [0]}}
+{"file_name": "0002.png", "objects": {"bbox": [[810.0, 100.0, 57.0, 28.0]], "categories": [1]}}
+{"file_name": "0003.png", "objects": {"bbox": [[160.0, 31.0, 248.0, 616.0], [741.0, 68.0, 202.0, 401.0]], "categories": [2, 2]}}
+```
+
+Load the dataset with `ImageFolder`, and it will create a `objects` column with the bounding boxes and the categories:
+
+```py
+>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder", split="train")
+>>> dataset[0]["objects"]
+{"bbox": [[302.0, 109.0, 73.0, 52.0]], "categories": [0]}
+```
+
+### Upload dataset to the Hub
+
+Once you've created a dataset, you can share it to the Hub with the [`~datasets.DatasetDict.push_to_hub`] method. Make sure you have the [huggingface_hub](https://huggingface.co/docs/huggingface_hub/index) library installed and you're logged in to your Hugging Face account (see the [Upload with Python tutorial](upload_dataset#upload-with-python) for more details).
+
+Upload your dataset with [`~datasets.DatasetDict.push_to_hub`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder", split="train")
+>>> dataset.push_to_hub("stevhliu/my-image-captioning-dataset")
+```
+
+## WebDataset
+
+The [WebDataset](https://github.com/webdataset/webdataset) format is based on TAR archives and is suitable for big image datasets.
+Indeed you can group your images in TAR archives (e.g. 1GB of images per TAR archive) and have thousands of TAR archives:
+
+```
+folder/train/00000.tar
+folder/train/00001.tar
+folder/train/00002.tar
+...
+```
+
+In the archives, each example is made of files sharing the same prefix:
+
+```
+e39871fd9fd74f55.jpg
+e39871fd9fd74f55.json
+f18b91585c4d3f3e.jpg
+f18b91585c4d3f3e.json
+ede6e66b2fb59aab.jpg
+ede6e66b2fb59aab.json
+ed600d57fcee4f94.jpg
+ed600d57fcee4f94.json
+...
+```
+
+You can put your images labels/captions/bounding boxes using JSON or text files for example.
+
+Load your WebDataset and it will create on column per file suffix (here "jpg" and "json"):
+
+```python
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("webdataset", data_dir="/path/to/folder", split="train")
+>>> dataset[0]["json"]
+{"bbox": [[302.0, 109.0, 73.0, 52.0]], "categories": [0]}
+```
+
+It's also possible to have several images per example like this:
+
+```
+e39871fd9fd74f55.input.jpg
+e39871fd9fd74f55.output.jpg
+e39871fd9fd74f55.json
+f18b91585c4d3f3e.input.jpg
+f18b91585c4d3f3e.output.jpg
+f18b91585c4d3f3e.json
+...
+```
+
+For more details on the WebDataset format and the python library, please check the [WebDataset documentation](https://webdataset.github.io/webdataset).
+
+## Lance
+
+[Lance](https://lance.org) is an open multimodal lakehouse table format. Lance tables can natively store not only text and scalar values,
+but also large binary objects (blobs) such as images, audio, and video alongside your tabular data.
+
+Starting from image files on disk plus associated metadata (for example, captions and dimensions), you can write a self-contained Lance dataset to a
+local `*.lance` directory. The resulting table can store your metadata columns alongside an `image` column containing the encoded image bytes.
+
+For example, you might start with metadata like:
+
+```text
+{'caption': 'Cordelia and Dudley on their wedding  day last year', 'height': 315, 'width': 233}
+{'caption': 'Statistics on challenges for automation in 2021', 'height': 299, 'width': 701}
+```
+
+You can define a `pyarrow` schema for your metadata and image bytes, build a table, and write it as a Lance dataset:
+
+```python
+import lance
+import pyarrow as pa
+
+schema = pa.schema(
+    [
+        pa.field("caption", pa.utf8()),
+        pa.field("height", pa.int32()),
+        pa.field("width", pa.int32()),
+        # ... add any additional metadata columns you want here ...
+        pa.field("image", pa.binary()),
+    ]
+)
+
+# Provide image files alongside metadata
+rows = [
+    {
+        "image_path": "/path/to/images/0001.jpg",
+        "caption": "Cordelia and Dudley on their wedding  day last year",
+        "height": 315,
+        "width": 233,
+    },
+    {
+        "image_path": "/path/to/images/0002.jpg",
+        "caption": "Statistics on challenges for automation in 2021",
+        "height": 299,
+        "width": 701,
+    },
+]
+
+image_bytes = []
+for r in rows:
+    with open(r["image_path"], "rb") as f:
+        image_bytes.append(f.read())
+
+table = pa.table(
+    {
+        "caption": [r["caption"] for r in rows],
+        "height": [r["height"] for r in rows],
+        "width": [r["width"] for r in rows],
+        "image": image_bytes,
+    },
+    schema=schema,
+)
+
+ds = lance.write_dataset(
+    table,
+    "./images.lance",
+    schema=schema,
+    mode="create",
+)
+```
+
+Here's a representative view of what a Lance table storing images might look like (the `image` column contains encoded bytes):
+
+```text
++-----------------------------------------------+--------+-------+-----+------------------------------+
+| caption                                       | height | width | ... | image                        |
++-----------------------------------------------+--------+-------+-----+------------------------------+
+| "Cordelia and Dudley on their wedding ..."    | 315    | 233   | ... | b"\\xff\\xd8\\xff...\\xd9"   |
+| "Statistics on challenges for automation ..." | 299    | 701   | ... | b"\\xff\\xd8\\xff...\\xd9"   |
++-----------------------------------------------+--------+-------+-----+------------------------------+
+```
+
+Using this approach, you can store arbitrarily large image datasets in Lance. The resulting `images.lance/` directory with
+its `*.lance` files can be uploaded to the Hugging Face Hub, just like the other examples above. See the `lance-format/laion-1m`
+[on the Hub](https://huggingface.co/datasets/lance-format/laion-1m) dataset for an example of a Lance image dataset.
+
+For more details on working with Lance datasets, see the [Lance documentation](https://lance.org).

docs/source/image_load.mdx

@@ -0,0 +1,188 @@
+# Load image data
+
+Image datasets have [`Image`] type columns, which contain PIL objects. 
+
+> [!TIP]
+> To work with image datasets, you need to have the `vision` dependency installed. Check out the [installation](./installation#vision) guide to learn how to install it.
+
+When you load an image dataset and call the image column, the images are decoded as PIL Images:
+
+```py
+>>> from datasets import load_dataset, Image
+
+>>> dataset = load_dataset("AI-Lab-Makerere/beans", split="train")
+>>> dataset[0]["image"]
+```
+
+> [!WARNING]
+> Index into an image dataset using the row index first and then the `image` column - `dataset[0]["image"]` - to avoid decoding and resampling all the image objects in the dataset. Otherwise, this can be a slow and time-consuming process if you have a large dataset.
+
+For a guide on how to load any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./loading">general loading guide</a>.
+
+## Local files
+
+You can load a dataset from the image path. Use the [`~Dataset.cast_column`] function to accept a column of image file paths, and decode it into a PIL image with the [`Image`] feature:
+```py
+>>> from datasets import Dataset, Image
+
+>>> dataset = Dataset.from_dict({"image": ["path/to/image_1", "path/to/image_2", ..., "path/to/image_n"]}).cast_column("image", Image())
+>>> dataset[0]["image"]
+<PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1200x215 at 0x15E6D7160>]
+```
+
+If you only want to load the underlying path to the image dataset without decoding the image object, set `decode=False` in the [`Image`] feature:
+
+```py
+>>> dataset = load_dataset("AI-Lab-Makerere/beans", split="train").cast_column("image", Image(decode=False))
+>>> dataset[0]["image"]
+{'bytes': None,
+ 'path': '/root/.cache/huggingface/datasets/downloads/extracted/b0a21163f78769a2cf11f58dfc767fb458fc7cea5c05dccc0144a2c0f0bc1292/train/bean_rust/bean_rust_train.29.jpg'}
+```
+
+## ImageFolder
+
+You can also load a dataset with an `ImageFolder` dataset builder which does not require writing a custom dataloader. This makes `ImageFolder` ideal for quickly creating and loading image datasets with several thousand images for different vision tasks. Your image dataset structure should look like this:
+
+```
+folder/train/dog/golden_retriever.png
+folder/train/dog/german_shepherd.png
+folder/train/dog/chihuahua.png
+
+folder/train/cat/maine_coon.png
+folder/train/cat/bengal.png
+folder/train/cat/birman.png
+```
+
+Alternatively it should have metadata, for example:
+
+```
+folder/train/metadata.csv
+folder/train/0001.png
+folder/train/0002.png
+folder/train/0003.png
+```
+
+If the dataset follows the `ImageFolder` structure, then you can load it directly with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_name")
+>>> # OR locally:
+>>> dataset = load_dataset("/path/to/folder")
+```
+
+For local datasets, this is equivalent to passing `imagefolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("imagefolder", data_dir="/path/to/folder")
+```
+
+Then you can access the videos as `PIL.Image` objects:
+
+```
+>>> dataset["train"][0]
+{"image": <PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1200x215 at 0x15E6D7160>, "label": 0}
+
+>>> dataset["train"][-1]
+{"image": <PIL.PngImagePlugin.PngImageFile image mode=RGBA size=1200x215 at 0x15E8DAD30>, "label": 1}
+```
+
+To ignore the information in the metadata file, set `drop_metadata=True` in [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_with_metadata", drop_metadata=True)
+```
+
+If you don't have a metadata file, `ImageFolder` automatically infers the label name from the directory name.
+If you want to drop automatically created labels, set `drop_labels=True`.
+In this case, your dataset will only contain an image column:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_without_metadata", drop_labels=True)
+```
+
+Finally the `filters` argument lets you load only a subset of the dataset, based on a condition on the label or the metadata. This is especially useful if the metadata is in Parquet format, since this format enables fast filtering. It is also recommended to use this argument with `streaming=True`, because by default the dataset is fully downloaded before filtering.
+
+```python
+>>> filters = [("label", "=", 0)]
+>>> dataset = load_dataset("username/dataset_name", streaming=True, filters=filters)
+```
+
+> [!TIP]
+> For more information about creating your own `ImageFolder` dataset, take a look at the [Create an image dataset](./image_dataset) guide.
+
+
+## WebDataset
+
+The [WebDataset](https://github.com/webdataset/webdataset) format is based on a folder of TAR archives and is suitable for big image datasets.
+Because of their size, WebDatasets are generally loaded in streaming mode (using `streaming=True`).
+
+You can load a WebDataset like this:
+
+```python
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("webdataset", data_dir="/path/to/folder", streaming=True)
+```
+
+## Lance
+
+[Lance](https://lance.org) is an open multimodal lakehouse table format. Lance tables can natively store not only text and scalar values,
+but also large binary objects (blobs) such as images, audio, and video alongside your tabular data.
+
+Lance keeps your metadata and image blobs together in one place, while still letting you efficiently scan only the metadata columns you care about
+without loading image bytes. When you're ready, you can fetch a small subset of rows (including the image blobs) and write them directly to files on
+your local filesystem.
+
+```python
+from datasets import load_dataset
+
+# Return as a Hugging Face dataset
+ds = load_dataset(
+    "lance-format/laion-1m",
+    split="train",
+    streaming=True
+)
+
+dir_name = "laion_samples"
+Path(dir_name).mkdir(exist_ok=True)
+
+for idx, row in enumerate(ds.take(3)):
+    with open(f"{dir_name}/{idx}.jpg", "wb") as f:
+        f.write(row["image"])
+```
+
+In this example, the `image` column contains the encoded image bytes, so you can write them directly to `.jpg` files.
+
+> [!NOTE] The `datasets` API doesn't currently push down operations to the Lance table, so for larger datasets it may be slow.
+> For now, you'll get much better performance using the `lance` Python package directly. See the
+> documentation on [the Hub](https://huggingface.co/docs/datasets-lance) for examples on usage.
+
+## Image decoding
+
+By default, images are decoded sequentially as `PIL.Images` when you iterate on a dataset.
+However it is possible to speed up the dataset significantly using multithreaded decoding:
+
+```python
+>>> import os
+>>> num_threads = num_threads = min(32, (os.cpu_count() or 1) + 4)
+>>> dataset = dataset.decode(num_threads=num_threads)
+>>> for example in dataset:  # up to 20 times faster !
+...     ...
+```
+
+You can enable multithreading using `num_threads`. This is especially useful to speed up remote data streaming.
+However it can be slower than `num_threads=0` for local data on fast disks.
+
+If you are not interested in the images decoded as `PIL.Images` and would like to access the path/bytes instead, you can disable decoding:
+
+```python
+>>> dataset = dataset.decode(False)
+```
+
+Note: [`IterableDataset.decode`] is only available for streaming datasets at the moment.

docs/source/image_process.mdx

@@ -0,0 +1,72 @@
+# Process image data
+
+This guide shows specific methods for processing image datasets. Learn how to:
+
+- Use [`~Dataset.map`] with image dataset.
+- Apply data augmentations to a dataset with [`~Dataset.set_transform`].
+
+For a guide on how to process any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./process">general process guide</a>.
+
+## Map
+
+The [`~Dataset.map`] function can apply transforms over an entire dataset.
+
+For example, create a basic [`Resize`](https://pytorch.org/vision/stable/generated/torchvision.transforms.Resize.html) function:
+
+```py
+>>> def transforms(examples):
+...     examples["pixel_values"] = [image.convert("RGB").resize((100,100)) for image in examples["image"]]
+...     return examples
+```
+
+Now use the [`~Dataset.map`] function to resize the entire dataset, and set `batched=True` to speed up the process by accepting batches of examples. The transform returns `pixel_values` as a cacheable `PIL.Image` object:
+
+```py
+>>> dataset = dataset.map(transforms, remove_columns=["image"], batched=True)
+>>> dataset[0]
+{'label': 6,
+ 'pixel_values': <PIL.PngImagePlugin.PngImageFile image mode=RGB size=100x100 at 0x7F058237BB10>}
+```
+
+The cache file saves time because you don't have to execute the same transform twice. The [`~Dataset.map`] function is best for operations you only run once per training - like resizing an image - instead of using it for operations executed for each epoch, like data augmentations.
+
+[`~Dataset.map`] takes up some memory, but you can reduce its memory requirements with the following parameters:
+
+- [`batch_size`](./package_reference/main_classes#datasets.DatasetDict.map.batch_size) determines the number of examples that are processed in one call to the transform function.
+- [`writer_batch_size`](./package_reference/main_classes#datasets.DatasetDict.map.writer_batch_size) determines the number of processed examples that are kept in memory before they are stored away.
+
+Both parameter values default to 1000, which can be expensive if you are storing images. Lower these values to use less memory when you use [`~Dataset.map`].
+
+## Apply transforms
+
+🤗 Datasets applies data augmentations from any library or package to your dataset. Transforms can be applied on-the-fly on batches of data with [`~Dataset.set_transform`], which consumes less disk space.
+
+> [!TIP]
+> The following example uses [torchvision](https://pytorch.org/vision/stable/index.html), but feel free to use other data augmentation libraries like [Albumentations](https://albumentations.ai/docs/), [Kornia](https://kornia.readthedocs.io/en/latest/), and [imgaug](https://imgaug.readthedocs.io/en/latest/).
+
+For example, if you'd like to change the color properties of an image randomly:
+
+```py
+>>> from torchvision.transforms import Compose, ColorJitter, ToTensor
+
+>>> jitter = Compose(
+...     [
+...          ColorJitter(brightness=0.25, contrast=0.25, saturation=0.25, hue=0.7),
+...          ToTensor(),
+...     ]
+... )
+```
+
+Create a function to apply the `ColorJitter` transform:
+
+```py
+>>> def transforms(examples):
+...     examples["pixel_values"] = [jitter(image.convert("RGB")) for image in examples["image"]]
+...     return examples
+```
+
+Apply the transform with the [`~Dataset.set_transform`] function:
+
+```py
+>>> dataset.set_transform(transforms)
+```

docs/source/index.mdx

@@ -0,0 +1,30 @@
+# Datasets
+
+<img class="float-left !m-0 !border-0 !dark:border-0 !shadow-none !max-w-lg w-[150px]" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/datasets_logo.png"/>
+
+🤗 Datasets is a library for easily accessing and sharing AI datasets for Audio, Computer Vision, and Natural Language Processing (NLP) tasks.
+
+Load a dataset in a single line of code, and use our powerful data processing and streaming methods to quickly get your dataset ready for training in a deep learning model. Backed by the Apache Arrow format, process large datasets with zero-copy reads without any memory constraints for optimal speed and efficiency. We also feature a deep integration with the [Hugging Face Hub](https://huggingface.co/datasets), allowing you to easily load and share a dataset with the wider machine learning community.
+
+Find your dataset today on the [Hugging Face Hub](https://huggingface.co/datasets), and take an in-depth look inside of it with the live viewer.
+
+<div class="mt-10">
+  <div class="w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-2 md:gap-y-4 md:gap-x-5">
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./tutorial"
+      ><div class="w-full text-center bg-gradient-to-br from-blue-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Tutorials</div>
+      <p class="text-gray-700">Learn the basics and become familiar with loading, accessing, and processing a dataset. Start here if you are using 🤗 Datasets for the first time!</p>
+    </a>
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./how_to"
+      ><div class="w-full text-center bg-gradient-to-br from-indigo-400 to-indigo-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">How-to guides</div>
+      <p class="text-gray-700">Practical guides to help you achieve a specific goal. Take a look at these guides to learn how to use 🤗 Datasets to solve real-world problems.</p>
+    </a>
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./about_arrow"
+      ><div class="w-full text-center bg-gradient-to-br from-pink-400 to-pink-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Conceptual guides</div>
+      <p class="text-gray-700">High-level explanations for building a better understanding about important topics such as the underlying data format, the cache, and how datasets are generated.</p>
+   </a>
+    <a class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg" href="./package_reference/main_classes"
+      ><div class="w-full text-center bg-gradient-to-br from-purple-400 to-purple-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">Reference</div>
+      <p class="text-gray-700">Technical descriptions of how 🤗 Datasets classes and methods work.</p>
+    </a>
+  </div>
+</div>

docs/source/installation.md

@@ -0,0 +1,103 @@
+# Installation
+
+Before you start, you'll need to setup your environment and install the appropriate packages. 🤗 Datasets is tested on **Python 3.10+**.
+
+> [!TIP]
+> If you want to use 🤗 Datasets with TensorFlow or PyTorch, you'll need to install them separately. Refer to the [TensorFlow installation page](https://www.tensorflow.org/install/pip#tensorflow-2-packages-are-available) or the [PyTorch installation page](https://pytorch.org/get-started/locally/#start-locally) for the specific install command for your framework.
+
+## Virtual environment
+
+You should install 🤗 Datasets in a [virtual environment](https://docs.python.org/3/library/venv.html) to keep things tidy and avoid dependency conflicts.
+
+1. Create and navigate to your project directory:
+
+   ```bash
+   mkdir ~/my-project
+   cd ~/my-project
+   ```
+
+2. Start a virtual environment inside your directory:
+
+   ```bash
+   python -m venv .env
+   ```
+
+3. Activate and deactivate the virtual environment with the following commands:
+
+   ```bash
+   # Activate the virtual environment
+   source .env/bin/activate
+
+   # Deactivate the virtual environment
+   source .env/bin/deactivate
+   ```
+
+Once you've created your virtual environment, you can install 🤗 Datasets in it.
+
+## pip
+
+The most straightforward way to install 🤗 Datasets is with pip:
+
+```bash
+pip install datasets
+```
+
+Run the following command to check if 🤗 Datasets has been properly installed:
+
+```bash
+python -c "from datasets import load_dataset; print(load_dataset('rajpurkar/squad', split='train')[0])"
+```
+
+This command downloads version 1 of the [Stanford Question Answering Dataset (SQuAD)](https://rajpurkar.github.io/SQuAD-explorer/), loads the training split, and prints the first training example. You should see:
+
+```python
+{'answers': {'answer_start': [515], 'text': ['Saint Bernadette Soubirous']}, 'context': 'Architecturally, the school has a Catholic character. Atop the Main Building\'s gold dome is a golden statue of the Virgin Mary. Immediately in front of the Main Building and facing it, is a copper statue of Christ with arms upraised with the legend "Venite Ad Me Omnes". Next to the Main Building is the Basilica of the Sacred Heart. Immediately behind the basilica is the Grotto, a Marian place of prayer and reflection. It is a replica of the grotto at Lourdes, France where the Virgin Mary reputedly appeared to Saint Bernadette Soubirous in 1858. At the end of the main drive (and in a direct line that connects through 3 statues and the Gold Dome), is a simple, modern stone statue of Mary.', 'id': '5733be284776f41900661182', 'question': 'To whom did the Virgin Mary allegedly appear in 1858 in Lourdes France?', 'title': 'University_of_Notre_Dame'}
+```
+
+## Audio
+
+To work with audio datasets, you need to install the [`Audio`] feature as an extra dependency:
+
+```bash
+pip install datasets[audio]
+```
+
+## Vision
+
+To work with image datasets, you need to install the [`Image`] feature as an extra dependency:
+
+```bash
+pip install datasets[vision]
+```
+
+## Mesh
+
+To work with mesh datasets, you need to install the [`Mesh`] feature as an extra dependency:
+
+```bash
+pip install datasets[mesh]
+```
+
+## source
+
+Building 🤗 Datasets from source lets you make changes to the code base. To install from the source, clone the repository and install with the following commands:
+
+```bash
+git clone https://github.com/huggingface/datasets.git
+cd datasets
+pip install -e .
+```
+
+Again, you can check if 🤗 Datasets was properly installed with the following command:
+
+```bash
+python -c "from datasets import load_dataset; print(load_dataset('rajpurkar/squad', split='train')[0])"
+```
+
+## conda
+
+🤗 Datasets can also be installed from conda, a package management system:
+
+```bash
+conda install -c huggingface -c conda-forge datasets
+```

docs/source/load_hub.mdx

@@ -0,0 +1,101 @@
+# Load a dataset from the Hub
+
+Finding high-quality datasets that are reproducible and accessible can be difficult. One of 🤗 Datasets main goals is to provide a simple way to load a dataset of any format or type. The easiest way to get started is to discover an existing dataset on the [Hugging Face Hub](https://huggingface.co/datasets) - a community-driven collection of datasets for tasks in NLP, computer vision, and audio - and use 🤗 Datasets to download and generate the dataset.
+
+This tutorial uses the [rotten_tomatoes](https://huggingface.co/datasets/rotten_tomatoes) and [MInDS-14](https://huggingface.co/datasets/PolyAI/minds14) datasets, but feel free to load any dataset you want and follow along. Head over to the Hub now and find a dataset for your task!
+
+## Load a dataset
+
+Before you take the time to download a dataset, it's often helpful to quickly get some general information about a dataset. A dataset's information is stored inside [`DatasetInfo`] and can include information such as the dataset description, features, and dataset size. 
+
+Use the [`load_dataset_builder`] function to load a dataset builder and inspect a dataset's attributes without committing to downloading it:
+
+```py
+>>> from datasets import load_dataset_builder
+>>> ds_builder = load_dataset_builder("cornell-movie-review-data/rotten_tomatoes")
+
+# Inspect dataset description
+>>> ds_builder.info.description
+Movie Review Dataset. This is a dataset of containing 5,331 positive and 5,331 negative processed sentences from Rotten Tomatoes movie reviews. This data was first used in Bo Pang and Lillian Lee, ``Seeing stars: Exploiting class relationships for sentiment categorization with respect to rating scales.'', Proceedings of the ACL, 2005.
+
+# Inspect dataset features
+>>> ds_builder.info.features
+{'label': ClassLabel(names=['neg', 'pos']),
+ 'text': Value('string')}
+```
+
+If you're happy with the dataset, then load it with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+```
+
+## Splits
+
+A split is a specific subset of a dataset like `train` and `test`. List a dataset's split names with the [`get_dataset_split_names`] function:
+
+```py
+>>> from datasets import get_dataset_split_names
+
+>>> get_dataset_split_names("cornell-movie-review-data/rotten_tomatoes")
+['train', 'validation', 'test']
+```
+
+Then you can load a specific split with the `split` parameter. Loading a dataset `split` returns a [`Dataset`] object:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+>>> dataset
+Dataset({
+    features: ['text', 'label'],
+    num_rows: 8530
+})
+```
+
+If you don't specify a `split`, 🤗 Datasets returns a [`DatasetDict`] object instead:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("cornell-movie-review-data/rotten_tomatoes")
+DatasetDict({
+    train: Dataset({
+        features: ['text', 'label'],
+        num_rows: 8530
+    })
+    validation: Dataset({
+        features: ['text', 'label'],
+        num_rows: 1066
+    })
+    test: Dataset({
+        features: ['text', 'label'],
+        num_rows: 1066
+    })
+})
+```
+
+## Configurations
+
+Some datasets contain several sub-datasets. For example, the [MInDS-14](https://huggingface.co/datasets/PolyAI/minds14) dataset has several sub-datasets, each one containing audio data in a different language. These sub-datasets are known as *configurations* or *subsets*, and you must explicitly select one when loading the dataset. If you don't provide a configuration name, 🤗 Datasets will raise a `ValueError` and remind you to choose a configuration.
+
+Use the [`get_dataset_config_names`] function to retrieve a list of all the possible configurations available to your dataset:
+
+```py
+>>> from datasets import get_dataset_config_names
+
+>>> configs = get_dataset_config_names("PolyAI/minds14")
+>>> print(configs)
+['cs-CZ', 'de-DE', 'en-AU', 'en-GB', 'en-US', 'es-ES', 'fr-FR', 'it-IT', 'ko-KR', 'nl-NL', 'pl-PL', 'pt-PT', 'ru-RU', 'zh-CN', 'all']
+```
+
+Then load the configuration you want:
+
+```py
+>>> from datasets import load_dataset
+
+>>> mindsFR = load_dataset("PolyAI/minds14", "fr-FR", split="train")
+```

docs/source/loading.mdx

@@ -0,0 +1,469 @@
+# Load
+
+Your data can be stored in various places; they can be on your local machine's disk, in a Github repository, and in in-memory data structures like Python dictionaries and Pandas DataFrames. Wherever a dataset is stored, 🤗 Datasets can help you load it.
+
+This guide will show you how to load a dataset from:
+
+- The Hugging Face Hub
+- Local files
+- In-memory data
+- Offline
+- A specific slice of a split
+
+For more details specific to loading other dataset modalities, take a look at the <a class="underline decoration-pink-400 decoration-2 font-semibold" href="./audio_load">load audio dataset guide</a>, the <a class="underline decoration-yellow-400 decoration-2 font-semibold" href="./image_load">load image dataset guide</a>, the <a class="underline decoration-blue-400 decoration-2 font-semibold" href="./video_load">load video dataset guide</a>, the <a class="underline decoration-purple-400 decoration-2 font-semibold" href="./mesh_load">load mesh dataset guide</a>, or the <a class="underline decoration-green-400 decoration-2 font-semibold" href="./nlp_load">load text dataset guide</a>.
+
+<a id='load-from-the-hub'></a>
+
+## Hugging Face Hub
+
+You can also load a dataset from any dataset repository on the Hub! Begin by [creating a dataset repository](share#create-the-repository) and upload your data files. Now you can use the [`load_dataset`] function to load the dataset. 
+
+For example, try loading the files from this [demo repository](https://huggingface.co/datasets/lhoestq/demo1) by providing the repository namespace and dataset name. This dataset repository contains CSV files, and the code below loads the dataset from the CSV files:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("lhoestq/demo1")
+```
+
+Some datasets may have more than one version based on Git tags, branches, or commits. Use the `revision` parameter to specify the dataset version you want to load:
+
+```py
+>>> dataset = load_dataset(
+...   "lhoestq/custom_squad",
+...   revision="main"  # tag name, or branch name, or commit hash
+... )
+```
+
+> [!TIP]
+> Refer to the [Upload a dataset to the Hub](./upload_dataset) tutorial for more details on how to create a dataset repository on the Hub, and how to upload your data files.
+
+A dataset loads by default all the data into the `train` split, or checks for mentions or split names in the data files names (e.g. "train", "test" and "validation"). Use the `data_files` parameter to map data files to splits like `train`, `validation` and `test`:
+
+```py
+>>> data_files = {"train": "train.csv", "test": "test.csv"}
+>>> dataset = load_dataset("namespace/your_dataset_name", data_files=data_files)
+```
+
+> [!WARNING]
+> If you don't specify which data files to use, [`load_dataset`] will return all the data files. This can take a long time if you load a large dataset like C4, which is approximately 13TB of data.
+
+You can also load a specific subset of the files with the `data_files` or `data_dir` parameter. These parameters can accept a relative path which resolves to the base path corresponding to where the dataset is loaded from.
+
+```py
+>>> from datasets import load_dataset
+
+# load files that match the grep pattern
+>>> c4_subset = load_dataset("allenai/c4", data_files="en/c4-train.0000*-of-01024.json.gz")
+
+# load dataset from the en directory on the Hub
+>>> c4_subset = load_dataset("allenai/c4", data_dir="en")
+```
+
+The `split` parameter can also map a data file to a specific split:
+
+```py
+>>> data_files = {"validation": "en/c4-validation.*.json.gz"}
+>>> c4_validation = load_dataset("allenai/c4", data_files=data_files, split="validation")
+```
+
+## Local and remote files
+
+Datasets can be loaded from local files stored on your computer and from remote files. The datasets are most likely stored as a `csv`, `json`, `txt`, `parquet` or `tsfile` file. The [`load_dataset`] function can load each of these file types.
+
+### CSV
+
+🤗 Datasets can read a dataset made up of one or several CSV files (in this case, pass your CSV files as a list):
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("csv", data_files="my_file.csv")
+```
+
+> [!TIP]
+> For more details, check out the [how to load tabular datasets from CSV files](tabular_load#csv-files) guide.
+
+### JSON
+
+JSON files are loaded directly with [`load_dataset`] as shown below:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("json", data_files="my_file.json")
+```
+
+JSON files have diverse formats, but we think the most efficient format is to have multiple JSON objects; each line represents an individual row of data. For example:
+
+```json
+{"a": 1, "b": 2.0, "c": "foo", "d": false}
+{"a": 4, "b": -5.5, "c": null, "d": true}
+```
+
+Another JSON format you may encounter is a nested field, in which case you'll need to specify the `field` argument as shown in the following:
+
+```py
+{"version": "0.1.0",
+ "data": [{"a": 1, "b": 2.0, "c": "foo", "d": false},
+          {"a": 4, "b": -5.5, "c": null, "d": true}]
+}
+
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("json", data_files="my_file.json", field="data")
+```
+
+To load remote JSON files via HTTP, pass the URLs instead:
+
+```py
+>>> base_url = "https://rajpurkar.github.io/SQuAD-explorer/dataset/"
+>>> dataset = load_dataset("json", data_files={"train": base_url + "train-v1.1.json", "validation": base_url + "dev-v1.1.json"}, field="data")
+```
+
+While these are the most common JSON formats, you'll see other datasets that are formatted differently. 🤗 Datasets recognizes these other formats and will fallback accordingly on the Python JSON loading methods to handle them.
+
+### Parquet
+
+Parquet files are stored in a columnar format, unlike row-based files like a CSV. Large datasets may be stored in a Parquet file because it is more efficient and faster at returning your query. 
+
+To load a Parquet file:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("parquet", data_files={'train': 'train.parquet', 'test': 'test.parquet'})
+```
+
+To load remote Parquet files via HTTP, pass the URLs instead:
+
+```py
+>>> base_url = "https://huggingface.co/datasets/wikimedia/wikipedia/resolve/main/20231101.ab/"
+>>> data_files = {"train": base_url + "train-00000-of-00001.parquet"}
+>>> wiki = load_dataset("parquet", data_files=data_files, split="train")
+```
+
+### Arrow
+
+Arrow files are stored in an in-memory columnar format, unlike row-based formats like CSV and uncompressed formats like Parquet.
+
+To load an Arrow file:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("arrow", data_files={'train': 'train.arrow', 'test': 'test.arrow'})
+```
+
+To load remote Arrow files via HTTP, pass the URLs instead:
+
+```py
+>>> base_url = "https://huggingface.co/datasets/croissantllm/croissant_dataset/resolve/main/english_660B_11/"
+>>> data_files = {"train": base_url + "train/data-00000-of-00080.arrow"}
+>>> wiki = load_dataset("arrow", data_files=data_files, split="train")
+```
+
+Arrow is the file format used by 🤗 Datasets under the hood, therefore you can load a local Arrow file using [`Dataset.from_file`] directly:
+
+```py
+>>> from datasets import Dataset
+>>> dataset = Dataset.from_file("data.arrow")
+```
+
+Unlike [`load_dataset`], [`Dataset.from_file`] memory maps the Arrow file without preparing the dataset in the cache, saving you disk space.
+The cache directory to store intermediate processing results will be the Arrow file directory in that case.
+
+For now only the Arrow streaming format is supported. The Arrow IPC file format (also known as Feather V2) is not supported.
+
+### Lance
+
+[Lance](https://lance.org) is an open multimodal lakehouse table format for AI. Lance tables can natively store not only text and scalar values, but also large binary objects (blobs) such as images, audio, and video alongside your tabular data.
+
+```py
+>>> from datasets import load_dataset
+>>> lance_base_url = "lance-format/laion-1m"
+```
+
+To stream the dataset without copying it to your local machine, specify the `streaming=True` parameter:
+
+```py
+ds = load_dataset(lance_base_url, split="train", streaming=True)
+# Take first three rows
+for row in ds.take(3):
+    print(row["caption"], row["image"])
+```
+
+This will return the image caption and the image bytes in a single request.
+
+### HDF5 files
+
+[HDF5](https://www.hdfgroup.org/solutions/hdf5/) files are commonly used for storing large amounts of numerical data in scientific computing and machine learning. Loading HDF5 files with 🤗 Datasets is similar to loading CSV files:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("hdf5", data_files="data.h5")
+```
+
+Note that the HDF5 loader assumes that the file has "tabular" structure, i.e. that all datasets in the file have (the same number of) rows on their first dimension.
+
+### TsFile
+
+[TsFile](https://tsfile.apache.org/) is a columnar file format designed for time-series data, used as the native storage layer of [Apache IoTDB](https://iotdb.apache.org/). It natively represents timestamps, device tags, and measurement fields, and maintains an internal time index that enables efficient time-range pruning.
+
+Each row in the resulting dataset corresponds to one **device** (identified by its TAG columns); the `time` column and every FIELD column are list columns containing that device's full time series, sorted in ascending time order.
+
+To load a TsFile:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("tsfile", data_files="my_data.tsfile")
+```
+
+Filter by time range — bounds are pushed down to TsFile's internal time index and accept `int` epochs, `datetime`, `date`, ISO-8601 strings, or `pyarrow` timestamp scalars:
+
+```py
+>>> from datetime import datetime
+>>> dataset = load_dataset(
+...     "tsfile",
+...     data_files="my_data.tsfile",
+...     start_time=datetime(2023, 11, 14),
+...     end_time=datetime(2023, 11, 15),
+... )
+```
+
+> [!TIP]
+> For more details, check out the [how to load TsFile data](tsfile_load) guide.
+
+### SQL
+
+Read database contents with [`~datasets.Dataset.from_sql`] by specifying the URI to connect to your database. You can read both table names and queries:
+
+```py
+>>> from datasets import Dataset
+# load entire table
+>>> dataset = Dataset.from_sql("data_table_name", con="sqlite:///sqlite_file.db")
+# load from query
+>>> dataset = Dataset.from_sql("SELECT text FROM table WHERE length(text) > 100 LIMIT 10", con="sqlite:///sqlite_file.db")
+```
+
+> [!TIP]
+> For more details, check out the [how to load tabular datasets from SQL databases](tabular_load#databases) guide.
+
+### WebDataset
+
+The [WebDataset](https://github.com/webdataset/webdataset) format is based on TAR archives and is suitable for big image datasets.
+Because of their size, WebDatasets are generally loaded in streaming mode (using `streaming=True`).
+
+You can load a WebDataset like this:
+
+```python
+>>> from datasets import load_dataset
+>>>
+>>> path = "path/to/train/*.tar"
+>>> dataset = load_dataset("webdataset", data_files={"train": path}, split="train", streaming=True)
+```
+
+To load remote WebDatasets via HTTP, pass the URLs instead:
+
+```python
+>>> from datasets import load_dataset
+>>>
+>>> base_url = "https://huggingface.co/datasets/lhoestq/small-publaynet-wds/resolve/main/publaynet-train-{i:06d}.tar"
+>>> urls = [base_url.format(i=i) for i in range(4)]
+>>> dataset = load_dataset("webdataset", data_files={"train": urls}, split="train", streaming=True)
+```
+
+## Remote files
+
+If you have remote files likely stored as a `csv`, `json`, `txt`, `parquet` or any supported format, the [`load_dataset`] function can load load them if you specify their remote paths:
+
+- `https://` URLs for public online files, e.g. `data_files=["https://rajpurkar.github.io/SQuAD-explorer/dataset/train-v2.0.json"]`
+- `hf://` URLs for files in any [Dataset repository](https://huggingface.co/docs/hub/datasets-overview) or [Storage Bucket](https://huggingface.co/docs/hub/storage-buckets) on Hugging Face, e.g. `data_files=["hf://datasets/karpathy/tinystories-gpt4-clean/tinystories_gpt4_clean.parquet"]` or `data_files=["hf://buckets/julien-c/my-training-bucket/julien/affluence.csv"]`
+
+## Multiprocessing
+
+When a dataset is made of several files (that we call "shards"), it is possible to significantly speed up the dataset downloading and preparation step.
+
+You can choose how many processes you'd like to use to prepare a dataset in parallel using `num_proc`.
+In this case, each process is given a subset of shards to prepare:
+
+```python
+from datasets import load_dataset
+
+imagenet = load_dataset("timm/imagenet-1k-wds", num_proc=8)
+ml_librispeech_spanish = load_dataset("facebook/multilingual_librispeech", "spanish", num_proc=8)
+```
+
+## In-memory data
+
+🤗 Datasets will also allow you to create a [`Dataset`] directly from in-memory data structures like Python dictionaries and Pandas DataFrames.
+
+### Python dictionary
+
+Load Python dictionaries with [`~Dataset.from_dict`]:
+
+```py
+>>> from datasets import Dataset
+>>> my_dict = {"a": [1, 2, 3]}
+>>> dataset = Dataset.from_dict(my_dict)
+```
+
+### Python list of dictionaries
+
+Load a list of Python dictionaries with [`~Dataset.from_list`]:
+
+```py
+>>> from datasets import Dataset
+>>> my_list = [{"a": 1}, {"a": 2}, {"a": 3}]
+>>> dataset = Dataset.from_list(my_list)
+```
+
+### Python generator
+
+Create a dataset from a Python generator with [`~Dataset.from_generator`]:
+
+```py
+>>> from datasets import Dataset
+>>> def my_gen():
+...     for i in range(1, 4):
+...         yield {"a": i}
+...
+>>> dataset = Dataset.from_generator(my_gen)
+```
+
+This approach supports loading data larger than available memory.
+
+You can also define a sharded dataset by passing lists to `gen_kwargs`:
+
+```py
+>>> def gen(shards):
+...     for shard in shards:
+...         with open(shard) as f:
+...             for line in f:
+...                 yield {"line": line}
+...
+>>> shards = [f"data{i}.txt" for i in range(32)]
+>>> ds = IterableDataset.from_generator(gen, gen_kwargs={"shards": shards})
+>>> ds = ds.shuffle(seed=42, buffer_size=10_000)  # shuffles the shards order + uses a shuffle buffer
+>>> from torch.utils.data import DataLoader
+>>> dataloader = DataLoader(ds.with_format("torch"), num_workers=4)  # give each worker a subset of 32/4=8 shards
+```
+
+### Pandas DataFrame
+
+Load Pandas DataFrames with [`~Dataset.from_pandas`]:
+
+```py
+>>> from datasets import Dataset
+>>> import pandas as pd
+>>> df = pd.DataFrame({"a": [1, 2, 3]})
+>>> dataset = Dataset.from_pandas(df)
+```
+
+> [!TIP]
+> For more details, check out the [how to load tabular datasets from Pandas DataFrames](tabular_load#pandas-dataframes) guide.
+
+## Offline
+
+Even if you don't have an internet connection, it is still possible to load a dataset. As long as you've downloaded a dataset from the Hub repository before, it should be cached. This means you can reload the dataset from the cache and use it offline.
+
+If you know you won't have internet access, you can run 🤗 Datasets in full offline mode. This saves time because instead of waiting for the Dataset builder download to time out, 🤗 Datasets will look directly in the cache. Set the environment variable `HF_HUB_OFFLINE` to `1` to enable full offline mode.
+
+## Slice splits
+
+You can also choose only to load specific slices of a split. There are two options for slicing a split: using strings or the [`ReadInstruction`] API. Strings are more compact and readable for simple cases, while [`ReadInstruction`] is easier to use with variable slicing parameters.
+
+Concatenate a `train` and `test` split by:
+
+```py
+>>> train_test_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train+test")
+===STRINGAPI-READINSTRUCTION-SPLIT===
+>>> ri = datasets.ReadInstruction("train") + datasets.ReadInstruction("test")
+>>> train_test_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split=ri)
+```
+
+Select specific rows of the `train` split:
+
+```py
+>>> train_10_20_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train[10:20]")
+===STRINGAPI-READINSTRUCTION-SPLIT===
+>>> train_10_20_ds = datasets.load_dataset("rojagtap/bookcorpus", split=datasets.ReadInstruction("train", from_=10, to=20, unit="abs"))
+```
+
+Or select a percentage of a split with:
+
+```py
+>>> train_10pct_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train[:10%]")
+===STRINGAPI-READINSTRUCTION-SPLIT===
+>>> train_10_20_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split=datasets.ReadInstruction("train", to=10, unit="%"))
+```
+
+Select a combination of percentages from each split:
+
+```py
+>>> train_10_80pct_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train[:10%]+train[-80%:]")
+===STRINGAPI-READINSTRUCTION-SPLIT===
+>>> ri = (datasets.ReadInstruction("train", to=10, unit="%") + datasets.ReadInstruction("train", from_=-80, unit="%"))
+>>> train_10_80pct_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split=ri)
+```
+
+Finally, you can even create cross-validated splits. The example below creates 10-fold cross-validated splits. Each validation dataset is a 10% chunk, and the training dataset makes up the remaining complementary 90% chunk:
+
+```py
+>>> val_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split=[f"train[{k}%:{k+10}%]" for k in range(0, 100, 10)])
+>>> train_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split=[f"train[:{k}%]+train[{k+10}%:]" for k in range(0, 100, 10)])
+===STRINGAPI-READINSTRUCTION-SPLIT===
+>>> val_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", [datasets.ReadInstruction("train", from_=k, to=k+10, unit="%") for k in range(0, 100, 10)])
+>>> train_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", [(datasets.ReadInstruction("train", to=k, unit="%") + datasets.ReadInstruction("train", from_=k+10, unit="%")) for k in range(0, 100, 10)])
+```
+
+### Percent slicing and rounding
+
+The default behavior is to round the boundaries to the nearest integer for datasets where the requested slice boundaries do not divide evenly by 100. As shown below, some slices may contain more examples than others. For instance, if the following train split includes 999 records, then:
+
+```py
+# 19 records, from 500 (included) to 519 (excluded).
+>>> train_50_52_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train[50%:52%]")
+# 20 records, from 519 (included) to 539 (excluded).
+>>> train_52_54_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train[52%:54%]")
+```
+
+If you want equal sized splits, use `pct1_dropremainder` rounding instead. This treats the specified percentage boundaries as multiples of 1%. 
+
+```py
+# 18 records, from 450 (included) to 468 (excluded).
+>>> train_50_52pct1_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split=datasets.ReadInstruction("train", from_=50, to=52, unit="%", rounding="pct1_dropremainder"))
+# 18 records, from 468 (included) to 486 (excluded).
+>>> train_52_54pct1_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split=datasets.ReadInstruction("train",from_=52, to=54, unit="%", rounding="pct1_dropremainder"))
+# Or equivalently:
+>>> train_50_52pct1_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train[50%:52%](pct1_dropremainder)")
+>>> train_52_54pct1_ds = datasets.load_dataset("ajibawa-2023/General-Stories-Collection", split="train[52%:54%](pct1_dropremainder)")
+```
+
+> [!WARNING]
+> `pct1_dropremainder` rounding may truncate the last examples in a dataset if the number of examples in your dataset don't divide evenly by 100.
+
+<a id='troubleshoot'></a>
+
+## Troubleshooting
+
+Sometimes, you may get unexpected results when you load a dataset. Two of the most common issues you may encounter are manually downloading a dataset and specifying features of a dataset.
+
+### Specify features
+
+When you create a dataset from local files, the [`Features`] are automatically inferred by [Apache Arrow](https://arrow.apache.org/docs/). However, the dataset's features may not always align with your expectations, or you may want to define the features yourself. The following example shows how you can add custom labels with the [`ClassLabel`] feature. 
+
+Start by defining your own labels with the [`Features`] class:
+
+```py
+>>> class_names = ["sadness", "joy", "love", "anger", "fear", "surprise"]
+>>> emotion_features = Features({'text': Value('string'), 'label': ClassLabel(names=class_names)})
+```
+
+Next, specify the `features` parameter in [`load_dataset`] with the features you just created:
+
+```py
+>>> dataset = load_dataset('csv', data_files=file_dict, delimiter=';', column_names=['text', 'label'], features=emotion_features)
+```
+
+Now when you look at your dataset features, you can see it uses the custom labels you defined:
+
+```py
+>>> dataset['train'].features
+{'text': Value('string'),
+'label': ClassLabel(names=['sadness', 'joy', 'love', 'anger', 'fear', 'surprise'])}
+```

docs/source/mesh_dataset.mdx

@@ -0,0 +1,164 @@
+# Create a mesh dataset
+
+There are two methods for creating and sharing a mesh dataset. This guide will show you how to:
+
+* Create a mesh dataset from local files in python with [`Dataset.push_to_hub`]. This is an easy way that requires only a few steps in python.
+
+* Create a mesh dataset with `MeshFolder` and some metadata. This is a no-code solution for quickly creating a mesh dataset with several thousand 3D files.
+
+> [!TIP]
+> You can control access to your dataset by requiring users to share their contact information first. Check out the [Gated datasets](https://huggingface.co/docs/hub/datasets-gated) guide for more information about how to enable this feature on the Hub.
+
+## Local files
+
+You can load your own dataset using the paths to your mesh files. Use the [`~Dataset.cast_column`] function to take a column of mesh file paths, and cast it to the [`Mesh`] feature:
+
+```py
+>>> from datasets import Dataset, Mesh
+
+>>> mesh_dataset = Dataset.from_dict({"mesh": ["path/to/model_1.glb", "path/to/model_2.ply", "path/to/model_3.stl"]}).cast_column("mesh", Mesh())
+>>> mesh_dataset[0]["mesh"]
+<trimesh.Scene(len(geometry)=33)>
+```
+
+Then upload the dataset to the Hugging Face Hub using [`Dataset.push_to_hub`]:
+
+```py
+mesh_dataset.push_to_hub("<username>/my_dataset")
+```
+
+This will create a dataset repository containing your mesh dataset:
+
+```text
+my_dataset/README.md
+my_dataset/data/train-00000-of-00001.parquet
+```
+
+## MeshFolder
+
+The `MeshFolder` is a dataset builder designed to quickly load a mesh dataset with several thousand mesh files without requiring you to write any code.
+
+> [!TIP]
+> Take a look at the [Split pattern hierarchy](repository_structure#split-pattern-hierarchy) to learn more about how `MeshFolder` creates dataset splits based on your dataset repository structure.
+
+`MeshFolder` automatically infers the class labels of your dataset based on the directory name. Store your dataset in a directory structure like:
+
+```text
+folder/train/diya/brass_diya.glb
+folder/train/diya/clay_diya.ply
+folder/train/diya/festival_diya.stl
+
+folder/train/kalash/copper_kalash.glb
+folder/train/kalash/pooja_kalash.ply
+folder/train/kalash/temple_kalash.stl
+```
+
+If the dataset follows the `MeshFolder` structure, then you can load it directly with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("path/to/folder")
+```
+
+This is equivalent to passing `meshfolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("meshfolder", data_dir="/path/to/folder")
+```
+
+You can also use `meshfolder` to load datasets involving multiple splits. To do so, your dataset directory should have the following structure:
+
+```text
+folder/train/diya/brass_diya.glb
+folder/train/kalash/copper_kalash.glb
+folder/test/diya/clay_diya.ply
+folder/test/kalash/temple_kalash.stl
+```
+
+> [!WARNING]
+> If all mesh files are contained in a single directory or if they are not on the same level of directory structure, `label` column won't be added automatically. If you need it, set `drop_labels=False` explicitly.
+
+If there is additional information you'd like to include about your dataset, like text captions or 3D asset metadata, add it as a `metadata.csv` file in your folder. You can also use a JSONL file `metadata.jsonl` or a Parquet file `metadata.parquet`.
+
+```text
+folder/train/metadata.csv
+folder/train/0001.glb
+folder/train/0002.ply
+folder/train/0003.stl
+```
+
+You can also zip your mesh files, and in this case each zip should contain both the mesh files and the metadata.
+
+```text
+folder/train.zip
+folder/test.zip
+folder/validation.zip
+```
+
+Your `metadata.csv` file must have a `file_name` or `*_file_name` field which links mesh files with their metadata:
+
+```csv
+file_name,caption
+0001.glb,A brass diya with a small bowl and raised wick holder
+0002.ply,A copper kalash with a rounded body and narrow neck
+0003.stl,A carved jharokha window with an arched frame
+```
+
+or using `metadata.jsonl`:
+
+```jsonl
+{"file_name": "0001.glb", "caption": "A brass diya with a small bowl and raised wick holder"}
+{"file_name": "0002.ply", "caption": "A copper kalash with a rounded body and narrow neck"}
+{"file_name": "0003.stl", "caption": "A carved jharokha window with an arched frame"}
+```
+
+Here the `file_name` must be the name of the mesh file next to the metadata file. More generally, it must be the relative path from the directory containing the metadata to the mesh file.
+
+It's possible to point to more than one mesh in each row in your dataset, for example if both your input and output are mesh files:
+
+```jsonl
+{"input_file_name": "0001.glb", "output_file_name": "0001_output.glb"}
+{"input_file_name": "0002.ply", "output_file_name": "0002_output.ply"}
+{"input_file_name": "0003.stl", "output_file_name": "0003_output.stl"}
+```
+
+You can also define lists of meshes. In that case you need to name the field `file_names` or `*_file_names`. Here is an example:
+
+```jsonl
+{"parts_file_names": ["0001_bowl.glb", "0001_wick_holder.glb"], "label": "diya"}
+{"parts_file_names": ["0002_body.ply", "0002_neck.ply"], "label": "kalash"}
+{"parts_file_names": ["0003_frame.stl", "0003_arch.stl"], "label": "jharokha"}
+```
+
+### Mesh captioning
+
+Mesh captioning datasets have text describing a 3D mesh. An example `metadata.csv` may look like:
+
+```csv
+file_name,text
+0001.glb,A brass diya with a small bowl and raised wick holder
+0002.ply,A copper kalash with a rounded body and narrow neck
+0003.stl,A carved jharokha window with an arched frame
+```
+
+Load the dataset with `MeshFolder`, and it will create a `text` column for the mesh captions:
+
+```py
+>>> dataset = load_dataset("meshfolder", data_dir="/path/to/folder", split="train")
+>>> dataset[0]["text"]
+"A brass diya with a small bowl and raised wick holder"
+```
+
+### Upload dataset to the Hub
+
+Once you've created a dataset, you can share it to the Hub with the [`~datasets.DatasetDict.push_to_hub`] method. Make sure you have the [huggingface_hub](https://huggingface.co/docs/huggingface_hub/index) library installed and you're logged in to your Hugging Face account (see the [Upload with Python tutorial](upload_dataset#upload-with-python) for more details).
+
+Upload your dataset with [`~datasets.DatasetDict.push_to_hub`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("meshfolder", data_dir="/path/to/folder", split="train")
+>>> dataset.push_to_hub("username/my-mesh-captioning-dataset")
+```

docs/source/mesh_load.mdx

@@ -0,0 +1,123 @@
+# Load mesh data
+
+Mesh datasets have [`Mesh`] type columns, which contain `trimesh.Trimesh` or `trimesh.Scene` objects.
+
+> [!TIP]
+> To work with mesh datasets, you need to have the `mesh` dependency installed. Check out the [installation](./installation#mesh) guide to learn how to install it.
+
+When you load a mesh dataset and call the mesh column, the meshes are decoded with `trimesh`:
+
+```py
+>>> from datasets import load_dataset, Mesh
+
+>>> dataset = load_dataset("VINAY-UMRETHE/My-Mesh-Dataset", split="train")
+>>> dataset[0]["mesh"]
+<trimesh.Scene(len(geometry)=33)>
+```
+
+Depending on the file content, `trimesh` may return a `trimesh.Trimesh` object or a `trimesh.Scene` object.
+
+> [!WARNING]
+> Index into a mesh dataset using the row index first and then the `mesh` column - `dataset[0]["mesh"]` - to avoid decoding all the mesh files in the dataset. Otherwise, this can be a slow and time-consuming process if you have a large dataset.
+
+For a guide on how to load any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./loading">general loading guide</a>.
+
+## Local files
+
+You can load a dataset from mesh paths. Use the [`~Dataset.cast_column`] function to accept a column of mesh file paths, and decode it into a `trimesh` object with the [`Mesh`] feature:
+
+```py
+>>> from datasets import Dataset, Mesh
+
+>>> dataset = Dataset.from_dict({"mesh": ["path/to/model_1.glb", "path/to/model_2.ply", "path/to/model_3.stl"]}).cast_column("mesh", Mesh())
+>>> dataset[0]["mesh"]
+<trimesh.Scene(len(geometry)=33)>
+```
+
+If you only want to load the underlying path or bytes of the mesh dataset without decoding the mesh object, set `decode=False` in the [`Mesh`] feature:
+
+```py
+>>> dataset = load_dataset("VINAY-UMRETHE/My-Mesh-Dataset", split="train").cast_column("mesh", Mesh(decode=False))
+>>> dataset[0]["mesh"]
+{'bytes': b'...',
+ 'path': '00001.glb'}
+```
+
+The [`Mesh`] feature supports `.glb`, `.ply`, and `.stl` files. Depending on the file content, `trimesh` may return a `trimesh.Trimesh` object or a `trimesh.Scene` object.
+
+## MeshFolder
+
+You can also load a dataset with a `MeshFolder` dataset builder which does not require writing a custom dataloader. This makes `MeshFolder` useful for quickly creating and loading mesh datasets with several thousand 3D files. Your mesh dataset structure should look like this:
+
+```text
+folder/train/diya/brass_diya.glb
+folder/train/diya/clay_diya.ply
+folder/train/diya/festival_diya.stl
+
+folder/train/kalash/copper_kalash.glb
+folder/train/kalash/pooja_kalash.ply
+folder/train/kalash/temple_kalash.stl
+```
+
+If the dataset follows the `MeshFolder` structure, then you can load it directly with [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_name")
+>>> # OR locally:
+>>> dataset = load_dataset("/path/to/folder")
+```
+
+For local datasets, this is equivalent to passing `meshfolder` manually in [`load_dataset`] and the directory in `data_dir`:
+
+```py
+>>> dataset = load_dataset("meshfolder", data_dir="/path/to/folder")
+```
+
+Then you can access the meshes as `trimesh` objects:
+
+```py
+>>> dataset["train"][0]
+{"mesh": <trimesh.Scene(len(geometry)=33)>, "label": 0}
+```
+
+To ignore the information in the metadata file, set `drop_metadata=True` in [`load_dataset`]:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_with_metadata", drop_metadata=True)
+```
+
+If you don't have a metadata file, `MeshFolder` automatically infers the label name from the directory name.
+If you want to drop automatically created labels, set `drop_labels=True`.
+In this case, your dataset will only contain a mesh column:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("username/dataset_without_metadata", drop_labels=True)
+```
+
+Finally the `filters` argument lets you load only a subset of the dataset, based on a condition on the label or the metadata. This is especially useful if the metadata is in Parquet format, since this format enables fast filtering. It is also recommended to use this argument with `streaming=True`, because by default the dataset is fully downloaded before filtering.
+
+```python
+>>> filters = [("label", "=", 0)]
+>>> dataset = load_dataset("username/dataset_name", streaming=True, filters=filters)
+```
+
+> [!TIP]
+> For more information about creating your own `MeshFolder` dataset, take a look at the [Create a mesh dataset](./mesh_dataset) guide.
+
+## Mesh decoding
+
+By default, meshes are decoded sequentially as `trimesh.Trimesh` or `trimesh.Scene` objects when you iterate on a dataset.
+
+If you are not interested in the meshes decoded as `trimesh` objects and would like to access the path or bytes instead, you can disable decoding:
+
+```python
+>>> dataset = dataset.decode(False)
+```
+
+Note: [`IterableDataset.decode`] is only available for streaming datasets at the moment.

docs/source/nifti_dataset.mdx

@@ -0,0 +1,130 @@
+# Create a NIfTI dataset
+
+This page shows how to create and share a dataset of medical images in NIfTI format (.nii / .nii.gz) using the `datasets` library.
+
+You can share a dataset with your team or with anyone in the community by creating a dataset repository on the Hugging Face Hub:
+
+```py
+from datasets import load_dataset
+
+dataset = load_dataset("<username>/my_nifti_dataset")
+```
+
+There are two common ways to create a NIfTI dataset:
+
+- Create a dataset from local NIfTI files in Python and upload it with `Dataset.push_to_hub`.
+- Use a folder-based convention (one file per example) and a small helper to convert it into a `Dataset`.
+
+> [!TIP]
+> You can control access to your dataset by requiring users to share their contact information first. Check out the [Gated datasets](https://huggingface.co/docs/hub/datasets-gated) guide for more information.
+
+## Local files
+
+If you already have a list of file paths to NIfTI files, the easiest workflow is to create a `Dataset` from that list and cast the column to the `Nifti` feature.
+
+```py
+from datasets import Dataset
+from datasets import Nifti
+
+# simple example: create a dataset from file paths
+files = ["/path/to/scan_001.nii.gz", "/path/to/scan_002.nii.gz"]
+ds = Dataset.from_dict({"nifti": files}).cast_column("nifti", Nifti())
+
+# access a decoded nibabel image (if decode=True)
+# ds[0]["nifti"] will be a nibabel.Nifti1Image object when decode=True
+# or a dict {'bytes': None, 'path': '...'} when decode=False
+```
+
+The `Nifti` feature supports a `decode` parameter. When `decode=True` (the default), it loads the NIfTI file into a `nibabel.nifti1.Nifti1Image` object. You can access the image data as a numpy array with `img.get_fdata()`. When `decode=False`, it returns a dict with the file path and bytes.
+
+```py
+from datasets import Dataset, Nifti
+
+ds = Dataset.from_dict({"nifti": ["/path/to/scan.nii.gz"]}).cast_column("nifti", Nifti(decode=True))
+img = ds[0]["nifti"]  # instance of: nibabel.nifti1.Nifti1Image
+arr = img.get_fdata()
+```
+
+After preparing the dataset you can push it to the Hub:
+
+```py
+ds.push_to_hub("<username>/my_nifti_dataset")
+```
+
+This will create a dataset repository containing your NIfTI dataset with a `data/` folder of parquet shards.
+
+## Folder conventions and metadata
+
+If you organize your dataset in folders you can create splits automatically (train/test/validation) by following a structure like:
+
+```
+dataset/train/scan_0001.nii
+dataset/train/scan_0002.nii
+dataset/validation/scan_1001.nii
+dataset/test/scan_2001.nii
+```
+
+If you have labels or other metadata, provide a `metadata.csv`, `metadata.jsonl`, or `metadata.parquet` in the folder so files can be linked to metadata rows. The metadata must contain a `file_name` (or `*_file_name`) field with the relative path to the NIfTI file next to the metadata file.
+
+Example `metadata.csv`:
+
+```csv
+file_name,patient_id,age,diagnosis
+scan_0001.nii.gz,P001,45,healthy
+scan_0002.nii.gz,P002,59,disease_x
+```
+
+The `Nifti` feature works with zipped datasets too — each zip can contain NIfTI files and a metadata file. This is useful when uploading large datasets as archives.
+This means your dataset structure could look like this (mixed compressed and uncompressed files):
+```
+dataset/train/scan_0001.nii.gz
+dataset/train/scan_0002.nii
+dataset/validation/scan_1001.nii.gz
+dataset/test/scan_2001.nii
+```
+
+## Converting to PyTorch tensors
+
+Use the [`~Dataset.set_transform`] function to apply the transformation on-the-fly to batches of the dataset:
+
+```py
+import torch 
+import nibabel
+import numpy as np
+
+def transform_to_pytorch(example):
+    example["nifti_torch"] = [torch.tensor(ex.get_fdata()) for ex in example["nifti"]]
+    return example
+
+ds.set_transform(transform_to_pytorch)
+
+```
+Accessing elements now (e.g. `ds[0]`) will yield torch tensors in the `"nifti_torch"` key.
+
+
+## Usage of NifTI1Image
+
+NifTI is a format to store the result of 3 (or even 4) dimensional brain scans. This includes 3 spatial dimensions (x,y,z)
+and optionally a time dimension (t). Furthermore, the given positions here are only relative to the scanner, therefore 
+the dimensions (4, 5, 6) are used to lift this to real world coordinates.
+
+You can visualize nifti files for instance leveraging `matplotlib` as follows:
+```python
+import matplotlib.pyplot as plt
+from datasets import load_dataset
+
+def show_slices(slices):
+   """ Function to display row of image slices """
+   fig, axes = plt.subplots(1, len(slices))
+   for i, slice in enumerate(slices):
+       axes[i].imshow(slice.T, cmap="gray", origin="lower")
+
+nifti_ds = load_dataset("<username>/my_nifti_dataset")
+for epi_img in nifti_ds:
+    nifti_img = epi_img["nifti"].get_fdata()
+    show_slices([nifti_img[:, :, 16], nifti_img[26, :, :], nifti_img[:, 30, :]])
+    plt.show()
+```
+
+For further reading we refer to the [nibabel documentation](https://nipy.org/nibabel/index.html) and especially [this nibabel tutorial](https://nipy.org/nibabel/coordinate_systems.html)
+---

docs/source/nlp_load.mdx

@@ -0,0 +1,46 @@
+# Load text data
+
+This guide shows you how to load text datasets. To learn how to load any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./loading">general loading guide</a>.
+
+Text files are one of the most common file types for storing a dataset. By default, 🤗 Datasets samples a text file line by line to build the dataset.
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("text", data_files={"train": ["my_text_1.txt", "my_text_2.txt"], "test": "my_test_file.txt"})
+
+# Load from a directory
+>>> dataset = load_dataset("text", data_dir="path/to/text/dataset")
+```
+
+To sample a text file by paragraph or even an entire document, use the `sample_by` parameter:
+
+```py
+# Sample by paragraph
+>>> dataset = load_dataset("text", data_files={"train": "my_train_file.txt", "test": "my_test_file.txt"}, sample_by="paragraph")
+
+# Sample by document
+>>> dataset = load_dataset("text", data_files={"train": "my_train_file.txt", "test": "my_test_file.txt"}, sample_by="document")
+```
+
+You can also use grep patterns to load specific files:
+
+```py
+>>> from datasets import load_dataset
+>>> c4_subset = load_dataset("allenai/c4", data_files="en/c4-train.0000*-of-01024.json.gz")
+```
+
+To load remote text files via HTTP, pass the URLs instead:
+
+```py
+>>> dataset = load_dataset("text", data_files="https://huggingface.co/datasets/hf-internal-testing/dataset_with_data_files/resolve/main/data/train.txt")
+```
+
+To load XML data you can use the "xml" loader, which is equivalent to "text" with sample_by="document":
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("xml", data_files={"train": ["my_xml_1.xml", "my_xml_2.xml"], "test": "my_xml_file.xml"})
+
+# Load from a directory
+>>> dataset = load_dataset("xml", data_dir="path/to/xml/dataset")
+```

docs/source/nlp_process.mdx

@@ -0,0 +1,63 @@
+# Process text data
+
+This guide shows specific methods for processing text datasets. Learn how to:
+
+- Tokenize a dataset with [`~Dataset.map`].
+- Align dataset labels with label ids for NLI datasets.
+
+For a guide on how to process any type of dataset, take a look at the <a class="underline decoration-sky-400 decoration-2 font-semibold" href="./process">general process guide</a>.
+
+## Map
+
+The [`~Dataset.map`] function supports processing batches of examples at once which speeds up tokenization.
+
+Load a tokenizer from 🤗 [Transformers](https://huggingface.co/transformers/):
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("bert-base-cased")
+```
+
+Set the `batched` parameter to `True` in the [`~Dataset.map`] function to apply the tokenizer to batches of examples:
+
+```py
+>>> dataset = dataset.map(lambda examples: tokenizer(examples["text"]), batched=True)
+>>> dataset[0]
+{'text': 'the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .', 
+ 'label': 1, 
+ 'input_ids': [101, 1996, 2600, 2003, 16036, 2000, 2022, 1996, 7398, 2301, 1005, 1055, 2047, 1000, 16608, 1000, 1998, 2008, 2002, 1005, 1055, 2183, 2000, 2191, 1037, 17624, 2130, 3618, 2084, 7779, 29058, 8625, 13327, 1010, 3744, 1011, 18856, 19513, 3158, 5477, 4168, 2030, 7112, 16562, 2140, 1012, 102], 
+ 'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0], 
+ 'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]}
+```
+
+The [`~Dataset.map`] function converts the returned values to a PyArrow-supported format. But explicitly returning the tensors as NumPy arrays is faster because it is a natively supported PyArrow format. Set `return_tensors="np"` when you tokenize your text:
+
+```py
+>>> dataset = dataset.map(lambda examples: tokenizer(examples["text"], return_tensors="np"), batched=True)
+```
+
+## Align
+
+The [`~Dataset.align_labels_with_mapping`] function aligns a dataset label id with the label name. Not all 🤗 Transformers models follow the prescribed label mapping of the original dataset, especially for NLI datasets. For example, the [MNLI](https://huggingface.co/datasets/glue) dataset uses the following label mapping:
+
+```py
+>>> label2id = {"entailment": 0, "neutral": 1, "contradiction": 2}
+```
+
+To align the dataset label mapping with the mapping used by a model, create a dictionary of the label name and id to align on:
+
+```py
+>>> label2id = {"contradiction": 0, "neutral": 1, "entailment": 2}
+```
+
+Pass the dictionary of the label mappings to the [`~Dataset.align_labels_with_mapping`] function, and the column to align on:
+
+```py
+>>> from datasets import load_dataset
+
+>>> mnli = load_dataset("nyu-mll/glue", "mnli", split="train")
+>>> mnli_aligned = mnli.align_labels_with_mapping(label2id, "label")
+```
+
+You can also use this function to assign a custom mapping of labels to ids.

docs/source/object_detection.mdx

@@ -0,0 +1,161 @@
+# Object detection
+
+Object detection models identify something in an image, and object detection datasets are used for applications such as autonomous driving and detecting natural hazards like wildfire. This guide will show you how to apply transformations to an object detection dataset following the [tutorial](https://albumentations.ai/docs/examples/example_bboxes/) from [Albumentations](https://albumentations.ai/docs/).
+
+To run these examples, make sure you have up-to-date versions of [albumentations](https://albumentations.ai/docs/) and [cv2](https://docs.opencv.org/4.10.0/) installed:
+
+```bash
+pip install -U albumentations opencv-python
+```
+
+In this example, you'll use the [`cppe-5`](https://huggingface.co/datasets/rishitdagli/cppe-5) dataset for identifying medical personal protective equipment (PPE) in the context of the COVID-19 pandemic.
+
+Load the dataset and take a look at an example:
+
+```py
+>>> from datasets import load_dataset
+
+>>> ds = load_dataset("rishitdagli/cppe-5")
+>>> example = ds['train'][0]
+>>> example
+{'height': 663,
+ 'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=943x663 at 0x7FC3DC756250>,
+ 'image_id': 15,
+ 'objects': {'area': [3796, 1596, 152768, 81002],
+  'bbox': [[302.0, 109.0, 73.0, 52.0],
+   [810.0, 100.0, 57.0, 28.0],
+   [160.0, 31.0, 248.0, 616.0],
+   [741.0, 68.0, 202.0, 401.0]],
+  'category': [4, 4, 0, 0],
+  'id': [114, 115, 116, 117]},
+ 'width': 943}
+```
+
+The dataset has the following fields:
+
+- `image`: PIL.Image.Image object containing the image.
+- `image_id`: The image ID.
+- `height`: The image height.
+- `width`: The image width.
+- `objects`: A dictionary containing bounding box metadata for the objects in the image:
+  - `id`: The annotation id.
+  - `area`: The area of the bounding box.
+  - `bbox`: The object's bounding box (in the [coco](https://albumentations.ai/docs/3-basic-usage/bounding-boxes-augmentations/#understanding-bounding-box-formats) format).
+  - `category`: The object's category, with possible values including `Coverall (0)`, `Face_Shield (1)`, `Gloves (2)`, `Goggles (3)` and `Mask (4)`.
+
+You can visualize the `bboxes` on the image using some internal torch utilities. To do that, you will need to reference the [`~datasets.ClassLabel`] feature associated with the category IDs so you can look up the string labels:
+
+
+```py
+>>> import torch
+>>> from torchvision.ops import box_convert
+>>> from torchvision.utils import draw_bounding_boxes
+>>> from torchvision.transforms.functional import pil_to_tensor, to_pil_image
+
+>>> categories = ds['train'].features['objects'].feature['category']
+
+>>> boxes_xywh = torch.tensor(example['objects']['bbox'])
+>>> boxes_xyxy = box_convert(boxes_xywh, 'xywh', 'xyxy')
+>>> labels = [categories.int2str(x) for x in example['objects']['category']]
+>>> to_pil_image(
+...     draw_bounding_boxes(
+...         pil_to_tensor(example['image']),
+...         boxes_xyxy,
+...         colors="red",
+...         labels=labels,
+...     )
+... )
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/visualize_detection_example.png"/>
+</div>
+
+
+With `albumentations`, you can apply transforms that will affect the image while also updating the `bboxes` accordingly. In this case, the image is resized to (480, 480), flipped horizontally, and brightened. 
+
+```py
+>>> import albumentations
+>>> import numpy as np
+
+>>> transform = albumentations.Compose([
+...     albumentations.Resize(480, 480),
+...     albumentations.HorizontalFlip(p=1.0),
+...     albumentations.RandomBrightnessContrast(p=1.0),
+... ], bbox_params=albumentations.BboxParams(format='coco',  label_fields=['category']))
+
+>>> image = np.array(example['image'])
+>>> out = transform(
+...     image=image,
+...     bboxes=example['objects']['bbox'],
+...     category=example['objects']['category'],
+... )
+```
+
+Now when you visualize the result, the image should be flipped, but the `bboxes` should still be in the right places.
+
+```py
+>>> image = torch.tensor(out['image']).permute(2, 0, 1)
+>>> boxes_xywh = torch.stack([torch.tensor(x) for x in out['bboxes']])
+>>> boxes_xyxy = box_convert(boxes_xywh, 'xywh', 'xyxy')
+>>> labels = [categories.int2str(x) for x in out['category']]
+>>> to_pil_image(
+...     draw_bounding_boxes(
+...         image,
+...         boxes_xyxy,
+...         colors='red',
+...         labels=labels
+...     )
+... )
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/visualize_detection_example_transformed.png"/>
+</div>
+
+Create a function to apply the transform to a batch of examples:
+
+```py
+>>> def transforms(examples):
+...     images, bboxes, categories = [], [], []
+...     for image, objects in zip(examples['image'], examples['objects']):
+...         image = np.array(image.convert("RGB"))
+...         out = transform(
+...             image=image,
+...             bboxes=objects['bbox'],
+...             category=objects['category']
+...         )
+...         images.append(torch.tensor(out['image']).permute(2, 0, 1))
+...         bboxes.append(torch.tensor(out['bboxes']))
+...         categories.append(out['category'])
+...     return {'image': images, 'bbox': bboxes, 'category': categories}
+```
+
+Use the [`~Dataset.set_transform`] function to apply the transform on-the-fly which consumes less disk space. The randomness of data augmentation may return a different image if you access the same example twice. It is especially useful when training a model for several epochs.
+
+```py
+>>> ds['train'].set_transform(transforms)
+```
+
+You can verify the transform works by visualizing the 10th example:
+
+```py
+>>> example = ds['train'][10]
+>>> to_pil_image(
+...     draw_bounding_boxes(
+...         example['image'],
+...         box_convert(example['bbox'], 'xywh', 'xyxy'),
+...         colors='red',
+...         labels=[categories.int2str(x) for x in example['category']]
+...     )
+... )
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/visualize_detection_example_transformed_2.png"/>
+</div>
+
+> [!TIP]
+> Now that you know how to process a dataset for object detection, learn
+> [how to train an object detection model](https://colab.research.google.com/github/NielsRogge/Transformers-Tutorials/blob/master/YOLOS/Fine_tuning_YOLOS_for_object_detection_on_custom_dataset_(balloon).ipynb)
+> and use it for inference.

docs/source/package_reference/builder_classes.mdx

@@ -0,0 +1,43 @@
+# Builder classes
+
+## Builders
+
+🤗 Datasets relies on two main classes during the dataset building process: [`DatasetBuilder`] and [`BuilderConfig`].
+
+[[autodoc]] datasets.DatasetBuilder
+
+[[autodoc]] datasets.GeneratorBasedBuilder
+
+[[autodoc]] datasets.ArrowBasedBuilder
+
+[[autodoc]] datasets.BuilderConfig
+
+## Download
+
+[[autodoc]] datasets.DownloadManager
+
+[[autodoc]] datasets.StreamingDownloadManager
+
+[[autodoc]] datasets.DownloadConfig
+
+[[autodoc]] datasets.DownloadMode
+
+## Verification
+
+[[autodoc]] datasets.VerificationMode
+
+## Splits
+
+[[autodoc]] datasets.SplitGenerator
+
+[[autodoc]] datasets.Split
+
+[[autodoc]] datasets.NamedSplit
+
+[[autodoc]] datasets.NamedSplitAll
+
+[[autodoc]] datasets.ReadInstruction
+
+## Version
+
+[[autodoc]] datasets.utils.Version

docs/source/package_reference/loading_methods.mdx

@@ -0,0 +1,120 @@
+# Loading methods
+
+Methods for listing and loading datasets:
+
+## Datasets
+
+[[autodoc]] datasets.load_dataset
+
+[[autodoc]] datasets.load_from_disk
+
+[[autodoc]] datasets.load_dataset_builder
+
+[[autodoc]] datasets.get_dataset_config_names
+
+[[autodoc]] datasets.get_dataset_infos
+
+[[autodoc]] datasets.get_dataset_split_names
+
+## From files
+
+Configurations used to load data files.
+They are used when loading local files or a dataset repository:
+
+- local files: `load_dataset("parquet", data_dir="path/to/data/dir")`
+- dataset repository: `load_dataset("allenai/c4")`
+
+You can pass arguments to `load_dataset` to configure data loading.
+For example you can specify the `sep` parameter to define the [`~datasets.packaged_modules.csv.CsvConfig`] that is used to load the data:
+
+```python
+load_dataset("csv", data_dir="path/to/data/dir", sep="\t")
+```
+
+### Text
+
+[[autodoc]] datasets.packaged_modules.text.TextConfig
+
+[[autodoc]] datasets.packaged_modules.text.Text
+
+### CSV
+
+[[autodoc]] datasets.packaged_modules.csv.CsvConfig
+
+[[autodoc]] datasets.packaged_modules.csv.Csv
+
+### JSON
+
+[[autodoc]] datasets.packaged_modules.json.JsonConfig
+
+[[autodoc]] datasets.packaged_modules.json.Json
+
+### XML
+
+[[autodoc]] datasets.packaged_modules.xml.XmlConfig
+
+[[autodoc]] datasets.packaged_modules.xml.Xml
+
+### Parquet
+
+[[autodoc]] datasets.packaged_modules.parquet.ParquetConfig
+
+[[autodoc]] datasets.packaged_modules.parquet.Parquet
+
+### Arrow
+
+[[autodoc]] datasets.packaged_modules.arrow.ArrowConfig
+
+[[autodoc]] datasets.packaged_modules.arrow.Arrow
+
+### SQL
+
+[[autodoc]] datasets.packaged_modules.sql.SqlConfig
+
+[[autodoc]] datasets.packaged_modules.sql.Sql
+
+### Images
+
+[[autodoc]] datasets.packaged_modules.imagefolder.ImageFolderConfig
+
+[[autodoc]] datasets.packaged_modules.imagefolder.ImageFolder
+
+### Audio
+
+[[autodoc]] datasets.packaged_modules.audiofolder.AudioFolderConfig
+
+[[autodoc]] datasets.packaged_modules.audiofolder.AudioFolder
+
+### Videos
+
+[[autodoc]] datasets.packaged_modules.videofolder.VideoFolderConfig
+
+[[autodoc]] datasets.packaged_modules.videofolder.VideoFolder
+
+### HDF5
+
+[[autodoc]] datasets.packaged_modules.hdf5.HDF5Config
+
+[[autodoc]] datasets.packaged_modules.hdf5.HDF5
+
+### TsFile
+
+[[autodoc]] datasets.packaged_modules.tsfile.TsFileConfig
+
+[[autodoc]] datasets.packaged_modules.tsfile.TsFile
+
+### Pdf
+
+[[autodoc]] datasets.packaged_modules.pdffolder.PdfFolderConfig
+
+[[autodoc]] datasets.packaged_modules.pdffolder.PdfFolder
+
+### Nifti
+
+[[autodoc]] datasets.packaged_modules.niftifolder.NiftiFolderConfig
+
+[[autodoc]] datasets.packaged_modules.niftifolder.NiftiFolder
+
+### WebDataset
+
+[[autodoc]] datasets.packaged_modules.webdataset.WebDataset

docs/source/package_reference/main_classes.mdx

@@ -0,0 +1,302 @@
+# Main classes
+
+
+## DatasetInfo
+
+[[autodoc]] datasets.DatasetInfo
+
+## Dataset
+
+The base class [`Dataset`] implements a Dataset backed by an Apache Arrow table.
+
+[[autodoc]] datasets.Dataset
+    - add_column
+    - add_item
+    - from_file
+    - from_buffer
+    - from_pandas
+    - from_dict
+    - from_list
+    - from_generator
+    - data
+    - cache_files
+    - num_columns
+    - num_rows
+    - column_names
+    - shape
+    - unique
+    - flatten
+    - cast
+    - cast_column
+    - remove_columns
+    - rename_column
+    - rename_columns
+    - select_columns
+    - class_encode_column
+    - __len__
+    - __iter__
+    - iter
+    - formatted_as
+    - set_format
+    - set_transform
+    - reset_format
+    - with_format
+    - with_transform
+    - __getitem__
+    - cleanup_cache_files
+    - map
+    - filter
+    - select
+    - sort
+    - shuffle
+    - skip
+    - take
+    - train_test_split
+    - shard
+    - repeat
+    - to_tf_dataset
+    - push_to_hub
+    - save_to_disk
+    - load_from_disk
+    - flatten_indices
+    - to_csv
+    - to_pandas
+    - to_dict
+    - to_json
+    - to_parquet
+    - to_sql
+    - to_iterable_dataset
+    - add_faiss_index
+    - add_faiss_index_from_external_arrays
+    - save_faiss_index
+    - load_faiss_index
+    - add_elasticsearch_index
+    - load_elasticsearch_index
+    - list_indexes
+    - get_index
+    - drop_index
+    - search
+    - search_batch
+    - get_nearest_examples
+    - get_nearest_examples_batch
+    - info
+    - split
+    - builder_name
+    - citation
+    - config_name
+    - dataset_size
+    - description
+    - download_checksums
+    - download_size
+    - features
+    - homepage
+    - license
+    - size_in_bytes
+    - supervised_keys
+    - version
+    - from_csv
+    - from_json
+    - from_parquet
+    - from_text
+    - from_sql
+    - align_labels_with_mapping
+
+[[autodoc]] datasets.concatenate_datasets
+
+[[autodoc]] datasets.interleave_datasets
+
+[[autodoc]] datasets.distributed.split_dataset_by_node
+
+[[autodoc]] datasets.enable_caching
+
+[[autodoc]] datasets.disable_caching
+
+[[autodoc]] datasets.is_caching_enabled
+
+[[autodoc]] datasets.Column
+
+## DatasetDict
+
+Dictionary with split names as keys ('train', 'test' for example), and `Dataset` objects as values.
+It also has dataset transform methods like map or filter, to process all the splits at once.
+
+[[autodoc]] datasets.DatasetDict
+    - data
+    - cache_files
+    - num_columns
+    - num_rows
+    - column_names
+    - shape
+    - unique
+    - cleanup_cache_files
+    - map
+    - filter
+    - sort
+    - shuffle
+    - set_format
+    - reset_format
+    - formatted_as
+    - with_format
+    - with_transform
+    - flatten
+    - cast
+    - cast_column
+    - remove_columns
+    - rename_column
+    - rename_columns
+    - select_columns
+    - class_encode_column
+    - push_to_hub
+    - save_to_disk
+    - load_from_disk
+    - from_csv
+    - from_json
+    - from_parquet
+    - from_text
+
+<a id='package_reference_features'></a>
+
+## IterableDataset
+
+The base class [`IterableDataset`] implements an iterable Dataset backed by python generators.
+
+[[autodoc]] datasets.IterableDataset
+    - from_file
+    - from_pandas
+    - from_dict
+    - from_list
+    - from_generator
+    - remove_columns
+    - select_columns
+    - cast_column
+    - cast
+    - decode
+    - __iter__
+    - iter
+    - map
+    - rename_column
+    - filter
+    - shuffle
+    - batch
+    - skip
+    - take
+    - shard
+    - reshard
+    - repeat
+    - to_csv
+    - to_pandas
+    - to_dict
+    - to_json
+    - to_parquet
+    - to_sql
+    - push_to_hub
+    - load_state_dict
+    - state_dict
+    - info
+    - split
+    - builder_name
+    - citation
+    - config_name
+    - dataset_size
+    - description
+    - download_checksums
+    - download_size
+    - features
+    - homepage
+    - license
+    - size_in_bytes
+    - supervised_keys
+    - version
+    - from_csv
+    - from_json
+    - from_parquet
+    - from_text
+
+[[autodoc]] datasets.IterableColumn
+
+## IterableDatasetDict
+
+Dictionary with split names as keys ('train', 'test' for example), and `IterableDataset` objects as values.
+
+[[autodoc]] datasets.IterableDatasetDict
+    - map
+    - filter
+    - shuffle
+    - with_format
+    - cast
+    - cast_column
+    - remove_columns
+    - rename_column
+    - rename_columns
+    - select_columns
+    - push_to_hub
+
+## Features
+
+[[autodoc]] datasets.Features
+
+### Scalar
+
+[[autodoc]] datasets.Value
+
+[[autodoc]] datasets.ClassLabel
+
+### Composite
+
+[[autodoc]] datasets.LargeList
+
+[[autodoc]] datasets.List
+
+[[autodoc]] datasets.Sequence
+
+### Translation
+
+[[autodoc]] datasets.Translation
+
+[[autodoc]] datasets.TranslationVariableLanguages
+
+### Arrays
+
+[[autodoc]] datasets.Array2D
+
+[[autodoc]] datasets.Array3D
+
+[[autodoc]] datasets.Array4D
+
+[[autodoc]] datasets.Array5D
+
+### Audio
+
+[[autodoc]] datasets.Audio
+
+### Image
+
+[[autodoc]] datasets.Image
+
+### Video
+
+[[autodoc]] datasets.Video
+
+### Mesh
+
+[[autodoc]] datasets.Mesh
+
+### Json
+
+[[autodoc]] datasets.Json
+
+### Pdf
+
+[[autodoc]] datasets.Pdf
+
+### Nifti
+
+[[autodoc]] datasets.Nifti
+
+## Filesystems
+
+[[autodoc]] datasets.filesystems.is_remote_filesystem
+
+## Fingerprint
+
+[[autodoc]] datasets.fingerprint.Hasher

docs/source/package_reference/table_classes.mdx

@@ -0,0 +1,138 @@
+# Table Classes
+
+Each `Dataset` object is backed by a PyArrow Table.
+A Table can be loaded from either the disk (memory mapped) or in memory.
+Several Table types are available, and they all inherit from [`table.Table`].
+
+## Table
+
+[[autodoc]] datasets.table.Table
+    - validate
+    - equals
+    - to_batches
+    - to_pydict
+    - to_pandas
+    - to_string
+    - field
+    - column
+    - itercolumns
+    - schema
+    - columns
+    - num_columns
+    - num_rows
+    - shape
+    - nbytes
+
+## InMemoryTable
+
+[[autodoc]] datasets.table.InMemoryTable
+    - validate
+    - equals
+    - to_batches
+    - to_pydict
+    - to_pandas
+    - to_string
+    - field
+    - column
+    - itercolumns
+    - schema
+    - columns
+    - num_columns
+    - num_rows
+    - shape
+    - nbytes
+    - column_names
+    - slice
+    - filter
+    - flatten
+    - combine_chunks
+    - cast
+    - replace_schema_metadata
+    - add_column
+    - append_column
+    - remove_column
+    - set_column
+    - rename_columns
+    - select
+    - drop
+    - from_file
+    - from_buffer
+    - from_pandas
+    - from_arrays
+    - from_pydict
+    - from_batches
+
+## MemoryMappedTable
+
+[[autodoc]] datasets.table.MemoryMappedTable
+    - validate
+    - equals
+    - to_batches
+    - to_pydict
+    - to_pandas
+    - to_string
+    - field
+    - column
+    - itercolumns
+    - schema
+    - columns
+    - num_columns
+    - num_rows
+    - shape
+    - nbytes
+    - column_names
+    - slice
+    - filter
+    - flatten
+    - combine_chunks
+    - cast
+    - replace_schema_metadata
+    - add_column
+    - append_column
+    - remove_column
+    - set_column
+    - rename_columns
+    - select
+    - drop
+    - from_file
+
+## ConcatenationTable
+
+[[autodoc]] datasets.table.ConcatenationTable
+    - validate
+    - equals
+    - to_batches
+    - to_pydict
+    - to_pandas
+    - to_string
+    - field
+    - column
+    - itercolumns
+    - schema
+    - columns
+    - num_columns
+    - num_rows
+    - shape
+    - nbytes
+    - column_names
+    - slice
+    - filter
+    - flatten
+    - combine_chunks
+    - cast
+    - replace_schema_metadata
+    - add_column
+    - append_column
+    - remove_column
+    - set_column
+    - rename_columns
+    - select
+    - drop
+    - from_blocks
+    - from_tables
+
+## Utils
+
+[[autodoc]] datasets.table.concat_tables
+
+[[autodoc]] datasets.table.list_table_cache_files

docs/source/package_reference/utilities.mdx

@@ -0,0 +1,58 @@
+# Utilities
+
+## Configure logging
+
+🤗 Datasets strives to be transparent and explicit about how it works, but this can be quite verbose at times. We have included a series of logging methods which allow you to easily adjust the level of verbosity of the entire library. Currently the default verbosity of the library is set to `WARNING`.
+
+To change the level of verbosity, use one of the direct setters. For instance, here is how to change the verbosity to the `INFO` level:
+
+```py
+import datasets
+datasets.logging.set_verbosity_info()
+```
+
+You can also use the environment variable `DATASETS_VERBOSITY` to override the default verbosity, and set it to one of the following: `debug`, `info`, `warning`, `error`, `critical`:
+
+```bash
+DATASETS_VERBOSITY=error ./myprogram.py
+```
+
+All the methods of this logging module are documented below. The main ones are:
+
+- [`logging.get_verbosity`] to get the current level of verbosity in the logger
+- [`logging.set_verbosity`] to set the verbosity to the level of your choice
+
+In order from the least to the most verbose (with their corresponding `int` values):
+
+1. `logging.CRITICAL` or `logging.FATAL` (int value, 50): only report the most critical errors.
+2. `logging.ERROR` (int value, 40): only report errors.
+3. `logging.WARNING` or `logging.WARN` (int value, 30): only reports error and warnings. This the default level used by the library.
+4. `logging.INFO` (int value, 20): reports error, warnings and basic information.
+5. `logging.DEBUG` (int value, 10): report all information.
+
+[[autodoc]] datasets.logging.get_verbosity
+
+[[autodoc]] datasets.logging.set_verbosity
+
+[[autodoc]] datasets.logging.set_verbosity_info
+
+[[autodoc]] datasets.logging.set_verbosity_warning
+
+[[autodoc]] datasets.logging.set_verbosity_debug
+
+[[autodoc]] datasets.logging.set_verbosity_error
+
+[[autodoc]] datasets.logging.disable_propagation
+
+[[autodoc]] datasets.logging.enable_propagation
+
+## Configure progress bars
+
+By default, `tqdm` progress bars will be displayed during dataset download and preprocessing. You can disable them globally by setting `HF_DATASETS_DISABLE_PROGRESS_BARS`
+environment variable. You can also enable/disable them using [`~utils.enable_progress_bars`] and [`~utils.disable_progress_bars`]. If set, the environment variable has priority on the helpers.
+
+[[autodoc]] datasets.utils.enable_progress_bars
+
+[[autodoc]] datasets.utils.disable_progress_bars
+
+[[autodoc]] datasets.utils.are_progress_bars_disabled

docs/source/process.mdx

@@ -0,0 +1,856 @@
+# Process
+
+🤗 Datasets provides many tools for modifying the structure and content of a dataset. These tools are important for tidying up a dataset, creating additional columns, converting between features and formats, and much more.
+
+This guide will show you how to:
+
+- Reorder rows and split the dataset.
+- Rename and remove columns, and other common column operations.
+- Apply processing functions to each example in a dataset.
+- Concatenate datasets.
+- Apply a custom formatting transform.
+- Save and export processed datasets.
+
+For more details specific to processing other dataset modalities, take a look at the <a class="underline decoration-pink-400 decoration-2 font-semibold" href="./audio_process">process audio dataset guide</a>, the <a class="underline decoration-yellow-400 decoration-2 font-semibold" href="./image_process">process image dataset guide</a>, or the <a class="underline decoration-green-400 decoration-2 font-semibold" href="./nlp_process">process text dataset guide</a>.
+
+The examples in this guide use the MRPC dataset, but feel free to load any dataset of your choice and follow along!
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("nyu-mll/glue", "mrpc", split="train")
+```
+
+> [!WARNING]
+> All processing methods in this guide return a new [`Dataset`] object. Modification is not done in-place. Be careful about overriding your previous dataset!
+
+## Sort, shuffle, select, split, and shard
+
+There are several functions for rearranging the structure of a dataset.
+These functions are useful for selecting only the rows you want, creating train and test splits, and sharding very large datasets into smaller chunks.
+
+### Sort
+
+Use [`~Dataset.sort`] to sort column values according to their numerical values. The provided column must be NumPy compatible.
+
+```py
+>>> dataset["label"][:10]
+[1, 0, 1, 0, 1, 1, 0, 1, 0, 0]
+>>> sorted_dataset = dataset.sort("label")
+>>> sorted_dataset["label"][:10]
+[0, 0, 0, 0, 0, 0, 0, 0, 0, 0]
+>>> sorted_dataset["label"][-10:]
+[1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+```
+
+Under the hood, this creates a list of indices that is sorted according to values of the column.
+This indices mapping is then used to access the right rows in the underlying Arrow table.
+
+### Shuffle
+
+The [`~Dataset.shuffle`] function randomly rearranges the column values. You can specify the `generator` parameter in this function to use a different `numpy.random.Generator` if you want more control over the algorithm used to shuffle the dataset.
+
+```py
+>>> shuffled_dataset = sorted_dataset.shuffle(seed=42)
+>>> shuffled_dataset["label"][:10]
+[1, 1, 1, 0, 1, 1, 1, 1, 1, 0]
+```
+
+Shuffling takes the list of indices `[0:len(my_dataset)]` and shuffles it to create an indices mapping.
+However as soon as your [`Dataset`] has an indices mapping, the speed can become 10x slower.
+This is because there is an extra step to get the row index to read using the indices mapping, and most importantly, you aren't reading contiguous chunks of data anymore.
+To restore the speed, you'd need to rewrite the entire dataset on your disk again using [`Dataset.flatten_indices`], which removes the indices mapping.
+Alternatively, you can switch to an [`IterableDataset`] and leverage its fast approximate shuffling [`IterableDataset.shuffle`]:
+
+```py
+>>> iterable_dataset = dataset.to_iterable_dataset(num_shards=128)
+>>> shuffled_iterable_dataset = iterable_dataset.shuffle(seed=42, buffer_size=1000)
+```
+
+### Select and Filter
+
+There are two options for filtering rows in a dataset: [`~Dataset.select`] and [`~Dataset.filter`].
+
+- [`~Dataset.select`] returns rows according to a list of indices:
+
+```py
+>>> small_dataset = dataset.select([0, 10, 20, 30, 40, 50])
+>>> len(small_dataset)
+6
+```
+
+- [`~Dataset.filter`] returns rows that match a specified condition:
+
+```py
+>>> start_with_ar = dataset.filter(lambda example: example["sentence1"].startswith("Ar"))
+>>> len(start_with_ar)
+6
+>>> start_with_ar["sentence1"]
+['Around 0335 GMT , Tab shares were up 19 cents , or 4.4 % , at A $ 4.56 , having earlier set a record high of A $ 4.57 .',
+'Arison said Mann may have been one of the pioneers of the world music movement and he had a deep love of Brazilian music .',
+'Arts helped coach the youth on an eighth-grade football team at Lombardi Middle School in Green Bay .',
+'Around 9 : 00 a.m. EDT ( 1300 GMT ) , the euro was at $ 1.1566 against the dollar , up 0.07 percent on the day .',
+"Arguing that the case was an isolated example , Canada has threatened a trade backlash if Tokyo 's ban is not justified on scientific grounds .",
+'Artists are worried the plan would harm those who need help most - performers who have a difficult time lining up shows .'
+]
+```
+
+[`~Dataset.filter`] can also filter by indices if you set `with_indices=True`:
+
+```py
+>>> even_dataset = dataset.filter(lambda example, idx: idx % 2 == 0, with_indices=True)
+>>> len(even_dataset)
+1834
+>>> len(dataset) / 2
+1834.0
+```
+
+Unless the list of indices to keep is contiguous, those methods also create an indices mapping under the hood.
+
+### Split
+
+The [`~Dataset.train_test_split`] function creates train and test splits if your dataset doesn't already have them. This allows you to adjust the relative proportions or an absolute number of samples in each split. In the example below, use the `test_size` parameter to create a test split that is 10% of the original dataset:
+
+```py
+>>> dataset.train_test_split(test_size=0.1)
+{'train': Dataset(schema: {'sentence1': 'string', 'sentence2': 'string', 'label': 'int64', 'idx': 'int32'}, num_rows: 3301),
+'test': Dataset(schema: {'sentence1': 'string', 'sentence2': 'string', 'label': 'int64', 'idx': 'int32'}, num_rows: 367)}
+>>> 0.1 * len(dataset)
+366.8
+```
+
+The splits are shuffled by default, but you can set `shuffle=False` to prevent shuffling.
+
+### Shard
+
+🤗 Datasets supports sharding to divide a very large dataset into a predefined number of chunks. Specify the `num_shards` parameter in [`~Dataset.shard`] to determine the number of shards to split the dataset into. You'll also need to provide the shard you want to return with the `index` parameter.
+
+For example, the [stanfordnlp/imdb](https://huggingface.co/datasets/stanfordnlp/imdb) dataset has 25000 examples:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("stanfordnlp/imdb", split="train")
+>>> print(dataset)
+Dataset({
+    features: ['text', 'label'],
+    num_rows: 25000
+})
+```
+
+After sharding the dataset into four chunks, the first shard will only have 6250 examples:
+
+```py
+>>> dataset.shard(num_shards=4, index=0)
+Dataset({
+    features: ['text', 'label'],
+    num_rows: 6250
+})
+>>> print(25000/4)
+6250.0
+```
+
+## Rename, remove, cast, and flatten
+
+The following functions allow you to modify the columns of a dataset. These functions are useful for renaming or removing columns, changing columns to a new set of features, and flattening nested column structures.
+
+### Rename
+
+Use [`~Dataset.rename_column`] when you need to rename a column in your dataset. Features associated with the original column are actually moved under the new column name, instead of just replacing the original column in-place.
+
+Provide [`~Dataset.rename_column`] with the name of the original column, and the new column name:
+
+```py
+>>> dataset
+Dataset({
+    features: ['sentence1', 'sentence2', 'label', 'idx'],
+    num_rows: 3668
+})
+>>> dataset = dataset.rename_column("sentence1", "sentenceA")
+>>> dataset = dataset.rename_column("sentence2", "sentenceB")
+>>> dataset
+Dataset({
+    features: ['sentenceA', 'sentenceB', 'label', 'idx'],
+    num_rows: 3668
+})
+```
+
+### Remove
+
+When you need to remove one or more columns, provide the column name to remove to the [`~Dataset.remove_columns`] function. Remove more than one column by providing a list of column names:
+
+```py
+>>> dataset = dataset.remove_columns("label")
+>>> dataset
+Dataset({
+    features: ['sentence1', 'sentence2', 'idx'],
+    num_rows: 3668
+})
+>>> dataset = dataset.remove_columns(["sentence1", "sentence2"])
+>>> dataset
+Dataset({
+    features: ['idx'],
+    num_rows: 3668
+})
+```
+
+Conversely, [`~Dataset.select_columns`] selects one or more columns to keep and removes the rest. This function takes either one or a list of column names:
+
+```py
+>>> dataset
+Dataset({
+    features: ['sentence1', 'sentence2', 'label', 'idx'],
+    num_rows: 3668
+})
+>>> dataset = dataset.select_columns(['sentence1', 'sentence2', 'idx'])
+>>> dataset
+Dataset({
+    features: ['sentence1', 'sentence2', 'idx'],
+    num_rows: 3668
+})
+>>> dataset = dataset.select_columns('idx')
+>>> dataset
+Dataset({
+    features: ['idx'],
+    num_rows: 3668
+})
+```
+
+### Cast
+
+The [`~Dataset.cast`] function transforms the feature type of one or more columns. This function accepts your new [`Features`] as its argument. The example below demonstrates how to change the [`ClassLabel`] and [`Value`] features:
+
+```py
+>>> dataset.features
+{'sentence1': Value('string'),
+'sentence2': Value('string'),
+'label': ClassLabel(names=['not_equivalent', 'equivalent']),
+'idx': Value('int32')}
+
+>>> from datasets import ClassLabel, Value
+>>> new_features = dataset.features.copy()
+>>> new_features["label"] = ClassLabel(names=["negative", "positive"])
+>>> new_features["idx"] = Value("int64")
+>>> dataset = dataset.cast(new_features)
+>>> dataset.features
+{'sentence1': Value('string'),
+'sentence2': Value('string'),
+'label': ClassLabel(names=['negative', 'positive']),
+'idx': Value('int64')}
+```
+
+> [!TIP]
+> Casting only works if the original feature type and new feature type are compatible. For example, you can cast a column with the feature type `Value("int32")` to `Value("bool")` if the original column only contains ones and zeros.
+
+Use the [`~Dataset.cast_column`] function to change the feature type of a single column. Pass the column name and its new feature type as arguments:
+
+```py
+>>> dataset.features
+{'audio': Audio(sampling_rate=44100, mono=True)}
+
+>>> dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
+>>> dataset.features
+{'audio': Audio(sampling_rate=16000, mono=True)}
+```
+
+### Flatten
+
+Sometimes a column can be a nested structure of several types. Take a look at the nested structure below from the SQuAD dataset:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("rajpurkar/squad", split="train")
+>>> dataset.features
+{'id': Value('string'),
+ 'title': Value('string'),
+ 'context': Value('string'),
+ 'question': Value('string'),
+ 'answers': {'text': List(Value('string')),
+  'answer_start': List(Value('int32'))}}
+```
+
+The `answers` field contains two subfields: `text` and `answer_start`. Use the [`~Dataset.flatten`] function to extract the subfields into their own separate columns:
+
+```py
+>>> flat_dataset = dataset.flatten()
+>>> flat_dataset
+Dataset({
+    features: ['id', 'title', 'context', 'question', 'answers.text', 'answers.answer_start'],
+ num_rows: 87599
+})
+```
+
+Notice how the subfields are now their own independent columns: `answers.text` and `answers.answer_start`.
+
+## Map
+
+Some of the more powerful applications of 🤗 Datasets come from using the [`~Dataset.map`] function. The primary purpose of [`~Dataset.map`] is to speed up processing functions. It allows you to apply a processing function to each example in a dataset, independently or in batches. This function can even create new rows and columns.
+
+In the following example, prefix each `sentence1` value in the dataset with `'My sentence: '`.
+
+Start by creating a function that adds `'My sentence: '` to the beginning of each sentence. The function needs to accept and output a `dict`:
+
+```py
+>>> def add_prefix(example):
+...     example["sentence1"] = 'My sentence: ' + example["sentence1"]
+...     return example
+```
+
+Now use [`~Dataset.map`] to apply the `add_prefix` function to the entire dataset:
+
+```py
+>>> updated_dataset = small_dataset.map(add_prefix)
+>>> updated_dataset["sentence1"][:5]
+['My sentence: Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
+"My sentence: Yucaipa owned Dominick 's before selling the chain to Safeway in 1998 for $ 2.5 billion .",
+'My sentence: They had published an advertisement on the Internet on June 10 , offering the cargo for sale , he added .',
+'My sentence: Around 0335 GMT , Tab shares were up 19 cents , or 4.4 % , at A $ 4.56 , having earlier set a record high of A $ 4.57 .',
+]
+```
+
+Let's take a look at another example, except this time, you'll remove a column with [`~Dataset.map`]. When you remove a column, it is only removed after the example has been provided to the mapped function. This allows the mapped function to use the content of the columns before they are removed.
+
+Specify the column to remove with the `remove_columns` parameter in [`~Dataset.map`]:
+
+```py
+>>> updated_dataset = dataset.map(lambda example: {"new_sentence": example["sentence1"]}, remove_columns=["sentence1"])
+>>> updated_dataset.column_names
+['sentence2', 'label', 'idx', 'new_sentence']
+```
+
+> [!TIP]
+> 🤗 Datasets also has a [`~Dataset.remove_columns`] function which is faster because it doesn't copy the data of the remaining columns.
+
+You can also use [`~Dataset.map`] with indices if you set `with_indices=True`. The example below adds the index to the beginning of each sentence:
+
+```py
+>>> updated_dataset = dataset.map(lambda example, idx: {"sentence2": f"{idx}: " + example["sentence2"]}, with_indices=True)
+>>> updated_dataset["sentence2"][:5]
+['0: Referring to him as only " the witness " , Amrozi accused his brother of deliberately distorting his evidence .',
+ "1: Yucaipa bought Dominick 's in 1995 for $ 693 million and sold it to Safeway for $ 1.8 billion in 1998 .",
+ "2: On June 10 , the ship 's owners had published an advertisement on the Internet , offering the explosives for sale .",
+ '3: Tab shares jumped 20 cents , or 4.6 % , to set a record closing high at A $ 4.57 .',
+ '4: PG & E Corp. shares jumped $ 1.63 or 8 percent to $ 21.03 on the New York Stock Exchange on Friday .'
+]
+```
+
+### Multiprocessing
+
+Multiprocessing significantly speeds up processing by parallelizing processes on the CPU. Set the `num_proc` parameter in [`~Dataset.map`] to set the number of processes to use:
+
+```py
+>>> updated_dataset = dataset.map(lambda example, idx: {"sentence2": f"{idx}: " + example["sentence2"]}, with_indices=True, num_proc=4)
+```
+
+The [`~Dataset.map`] also works with the rank of the process if you set `with_rank=True`. This is analogous to the `with_indices` parameter. The `with_rank` parameter in the mapped function goes after the `index` one if it is already present.
+
+```py
+>>> import torch
+>>> from multiprocess import set_start_method
+>>> from transformers import AutoTokenizer, AutoModelForCausalLM
+>>> from datasets import load_dataset
+>>>
+>>> # Get an example dataset
+>>> dataset = load_dataset("fka/awesome-chatgpt-prompts", split="train")
+>>>
+>>> # Get an example model and its tokenizer
+>>> model = AutoModelForCausalLM.from_pretrained("Qwen/Qwen1.5-0.5B-Chat").eval()
+>>> tokenizer = AutoTokenizer.from_pretrained("Qwen/Qwen1.5-0.5B-Chat")
+>>>
+>>> def gpu_computation(batch, rank):
+...     # Move the model on the right GPU if it's not there already
+...     device = f"cuda:{(rank or 0) % torch.cuda.device_count()}"
+...     model.to(device)
+...
+...     # Your big GPU call goes here, for example:
+...     chats = [[
+...         {"role": "system", "content": "You are a helpful assistant."},
+...         {"role": "user", "content": prompt}
+...     ] for prompt in batch["prompt"]]
+...     texts = [tokenizer.apply_chat_template(
+...         chat,
+...         tokenize=False,
+...         add_generation_prompt=True
+...     ) for chat in chats]
+...     model_inputs = tokenizer(texts, padding=True, return_tensors="pt").to(device)
+...     with torch.no_grad():
+...         outputs = model.generate(**model_inputs, max_new_tokens=512)
+...     batch["output"] = tokenizer.batch_decode(outputs, skip_special_tokens=True)
+...     return batch
+>>>
+>>> if __name__ == "__main__":
+...     set_start_method("spawn")
+...     updated_dataset = dataset.map(
+...         gpu_computation,
+...         batched=True,
+...         batch_size=16,
+...         with_rank=True,
+...         num_proc=torch.cuda.device_count(),  # one process per GPU
+...     )
+```
+
+The main use-case for rank is to parallelize computation across several GPUs. This requires setting `multiprocess.set_start_method("spawn")`. If you don't you'll receive the following CUDA error:
+
+```bash
+RuntimeError: Cannot re-initialize CUDA in forked subprocess. To use CUDA with multiprocessing, you must use the 'spawn' start method.
+```
+
+### Batch processing
+
+The [`~Dataset.map`] function supports working with batches of examples. Operate on batches by setting `batched=True`. The default batch size is 1000, but you can adjust it with the `batch_size` parameter. Batch processing enables interesting applications such as splitting long sentences into shorter chunks and data augmentation.
+
+#### Split long examples
+
+When examples are too long, you may want to split them into several smaller chunks. Begin by creating a function that:
+
+1. Splits the `sentence1` field into chunks of 50 characters.
+
+2. Stacks all the chunks together to create the new dataset.
+
+```py
+>>> def chunk_examples(examples):
+...     chunks = []
+...     for sentence in examples["sentence1"]:
+...         chunks += [sentence[i:i + 50] for i in range(0, len(sentence), 50)]
+...     return {"chunks": chunks}
+```
+
+Apply the function with [`~Dataset.map`]:
+
+```py
+>>> chunked_dataset = dataset.map(chunk_examples, batched=True, remove_columns=dataset.column_names)
+>>> chunked_dataset[:10]
+{'chunks': ['Amrozi accused his brother , whom he called " the ',
+            'witness " , of deliberately distorting his evidenc',
+            'e .',
+            "Yucaipa owned Dominick 's before selling the chain",
+            ' to Safeway in 1998 for $ 2.5 billion .',
+            'They had published an advertisement on the Interne',
+            't on June 10 , offering the cargo for sale , he ad',
+            'ded .',
+            'Around 0335 GMT , Tab shares were up 19 cents , or',
+            ' 4.4 % , at A $ 4.56 , having earlier set a record']}
+```
+
+Notice how the sentences are split into shorter chunks now, and there are more rows in the dataset.
+
+```py
+>>> dataset
+Dataset({
+ features: ['sentence1', 'sentence2', 'label', 'idx'],
+ num_rows: 3668
+})
+>>> chunked_dataset
+Dataset({
+    features: ['chunks'],
+    num_rows: 10470
+})
+```
+
+#### Data augmentation
+
+The [`~Dataset.map`] function could also be used for data augmentation. The following example generates additional words for a masked token in a sentence.
+
+Load and use the [RoBERTA](https://huggingface.co/roberta-base) model in 🤗 Transformers' [FillMaskPipeline](https://huggingface.co/transformers/main_classes/pipelines#transformers.FillMaskPipeline):
+
+```py
+>>> from random import randint
+>>> from transformers import pipeline
+
+>>> fillmask = pipeline("fill-mask", model="roberta-base")
+>>> mask_token = fillmask.tokenizer.mask_token
+>>> smaller_dataset = dataset.filter(lambda e, i: i<100, with_indices=True)
+```
+
+Create a function to randomly select a word to mask in the sentence. The function should also return the original sentence and the top two replacements generated by RoBERTA.
+
+```py
+>>> def augment_data(examples):
+...     outputs = []
+...     for sentence in examples["sentence1"]:
+...         words = sentence.split(' ')
+...         K = randint(1, len(words)-1)
+...         masked_sentence = " ".join(words[:K]  + [mask_token] + words[K+1:])
+...         predictions = fillmask(masked_sentence)
+...         augmented_sequences = [predictions[i]["sequence"] for i in range(3)]
+...         outputs += [sentence] + augmented_sequences
+...
+...     return {"data": outputs}
+```
+
+Use [`~Dataset.map`] to apply the function over the whole dataset:
+
+```py
+>>> augmented_dataset = smaller_dataset.map(augment_data, batched=True, remove_columns=dataset.column_names, batch_size=8)
+>>> augmented_dataset[:9]["data"]
+['Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
+ 'Amrozi accused his brother, whom he called " the witness ", of deliberately withholding his evidence.',
+ 'Amrozi accused his brother, whom he called " the witness ", of deliberately suppressing his evidence.',
+ 'Amrozi accused his brother, whom he called " the witness ", of deliberately destroying his evidence.',
+ "Yucaipa owned Dominick 's before selling the chain to Safeway in 1998 for $ 2.5 billion .",
+ 'Yucaipa owned Dominick Stores before selling the chain to Safeway in 1998 for $ 2.5 billion.',
+ "Yucaipa owned Dominick's before selling the chain to Safeway in 1998 for $ 2.5 billion.",
+ 'Yucaipa owned Dominick Pizza before selling the chain to Safeway in 1998 for $ 2.5 billion.'
+]
+```
+
+For each original sentence, RoBERTA augmented a random word with three alternatives. The original word `distorting` is supplemented by `withholding`, `suppressing`, and `destroying`.
+
+### Asynchronous processing
+
+Asynchronous functions are useful to call API endpoints in parallel, for example to download content like images or call a model endpoint.
+
+You can define an asynchronous function using the `async` and `await` keywords, here is an example function to call a chat model from Hugging Face:
+
+```python
+>>> import aiohttp
+>>> import asyncio
+>>> from huggingface_hub import get_token
+>>> sem = asyncio.Semaphore(20)  # max number of simultaneous queries
+>>> async def query_model(model, prompt):
+...     api_url = f"https://api-inference.huggingface.co/models/{model}/v1/chat/completions"
+...     headers = {"Authorization": f"Bearer {get_token()}", "Content-Type": "application/json"}
+...     json = {"messages": [{"role": "user", "content": prompt}], "max_tokens": 20, "seed": 42}
+...     async with sem, aiohttp.ClientSession() as session, session.post(api_url, headers=headers, json=json) as response:
+...         output = await response.json()
+...         return {"Output": output["choices"][0]["message"]["content"]}
+```
+
+Asynchronous functions run in parallel, which accelerates the process a lot. The same code takes a lot more time if it's run sequentially, because it does nothing while waiting for the model response. It is generally recommended to use `async` / `await` when you function has to wait for a response from an API for example, or if it downloads data and it can take some time.
+
+Note the presence of a `Semaphore`: it sets the maximum number of queries that can run in parallel. It is recommended to use a `Semaphore` when calling APIs to avoid rate limit errors.
+
+Let's use it to call the [microsoft/Phi-3-mini-4k-instruct](https://huggingface.co/microsoft/Phi-3-mini-4k-instruct) model and ask it to return the main topic of each math problem in the [Maxwell-Jia/AIME_2024](https://huggingface.co/Maxwell-Jia/AIME_2024) dataset:
+
+````python
+>>> from datasets import load_dataset
+>>> ds = load_dataset("Maxwell-Jia/AIME_2024", split="train")
+>>> model = "microsoft/Phi-3-mini-4k-instruct"
+>>> prompt = 'What is this text mainly about ? Here is the text:\n\n```\n{Problem}\n```\n\nReply using one or two words max, e.g. "The main topic is Linear Algebra".'
+>>> async def get_topic(example):
+...     return await query_model(model, prompt.format(Problem=example['Problem']))
+>>> ds = ds.map(get_topic)
+>>> ds[0]
+{'ID': '2024-II-4',
+ 'Problem': 'Let $x,y$ and $z$ be positive real numbers that...',
+ 'Solution': 'Denote $\\log_2(x) = a$, $\\log_2(y) = b$, and...,
+ 'Answer': 33,
+ 'Output': 'The main topic is Logarithms.'}
+````
+
+Here, [`Dataset.map`] runs many `get_topic` function asynchronously so it doesn't have to wait for every single model response which would take a lot of time to do sequentially.
+
+By default, [`Dataset.map`] runs up to one thousand map functions in parallel, so don't forget to set the maximum number of API calls that can run in parallel with a `Semaphore`, otherwise the model could return rate limit errors or overload. For advanced use cases, you can change the maximum number of queries in parallel in `datasets.config`.
+
+### Process multiple splits
+
+Many datasets have splits that can be processed simultaneously with [`DatasetDict.map`]. For example, tokenize the `sentence1` field in the train and test split by:
+
+```py
+>>> from datasets import load_dataset
+
+# load all the splits
+>>> dataset = load_dataset('nyu-mll/glue', 'mrpc')
+>>> encoded_dataset = dataset.map(lambda examples: tokenizer(examples["sentence1"]), batched=True)
+>>> encoded_dataset["train"][0]
+{'sentence1': 'Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
+'sentence2': 'Referring to him as only " the witness " , Amrozi accused his brother of deliberately distorting his evidence .',
+'label': 1,
+'idx': 0,
+'input_ids': [  101,  7277,  2180,  5303,  4806,  1117,  1711,   117,  2292, 1119,  1270,   107,  1103,  7737,   107,   117,  1104,  9938, 4267, 12223, 21811,  1117,  2554,   119,   102],
+'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0],
+'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1]
+}
+```
+
+### Distributed usage
+
+When you use [`~Dataset.map`] in a distributed setting, you should also use [torch.distributed.barrier](https://pytorch.org/docs/stable/distributed?highlight=barrier#torch.distributed.barrier). This ensures the main process performs the mapping, while the other processes load the results, thereby avoiding duplicate work.
+
+The following example shows how you can use `torch.distributed.barrier` to synchronize the processes:
+
+```py
+>>> from datasets import Dataset
+>>> import torch.distributed
+
+>>> dataset1 = Dataset.from_dict({"a": [0, 1, 2]})
+
+>>> if training_args.local_rank > 0:
+...     print("Waiting for main process to perform the mapping")
+...     torch.distributed.barrier()
+
+>>> dataset2 = dataset1.map(lambda x: {"a": x["a"] + 1})
+
+>>> if training_args.local_rank == 0:
+...     print("Loading results from main process")
+...     torch.distributed.barrier()
+```
+
+## Batch
+
+The [`~Dataset.batch`] method allows you to group samples from the dataset into batches. This is particularly useful when you want to create batches of data for training or evaluation, especially when working with deep learning models.
+
+Here's an example of how to use the `batch()` method:
+
+```python
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("cornell-movie-review-data/rotten_tomatoes", split="train")
+>>> batched_dataset = dataset.batch(batch_size=4)
+>>> batched_dataset[0]
+{'text': ['the rock is destined to be the 21st century\'s new " conan " and that he\'s going to make a splash even greater than arnold schwarzenegger , jean-claud van damme or steven segal .',
+        'the gorgeously elaborate continuation of " the lord of the rings " trilogy is so huge that a column of words cannot adequately describe co-writer/director peter jackson\'s expanded vision of j . r . r . tolkien\'s middle-earth .',
+        'effective but too-tepid biopic',
+        'if you sometimes like to go to the movies to have fun , wasabi is a good place to start .'],
+'label': [1, 1, 1, 1]}
+```
+
+The `batch()` method accepts the following parameters:
+
+- `batch_size` (`int`): The number of samples in each batch.
+- `drop_last_batch` (`bool`, defaults to `False`): Whether to drop the last incomplete batch if the dataset size is not divisible by the batch size.
+- `num_proc` (`int`, optional, defaults to `None`): The number of processes to use for multiprocessing. If None, no multiprocessing is used. This can significantly speed up batching for large datasets.
+
+Note that `Dataset.batch()` returns a new [`Dataset`] where each item is a batch of multiple samples from the original dataset. If you want to process data in batches, you should use a batched [`~Dataset.map`] directly, which applies a function to batches but the output dataset is unbatched.
+
+## Concatenate
+
+Separate datasets can be concatenated if they share the same column types. Concatenate datasets with [`concatenate_datasets`]:
+
+```py
+>>> from datasets import concatenate_datasets, load_dataset
+
+>>> stories = load_dataset("ajibawa-2023/General-Stories-Collection", split="train")
+>>> stories = stories.select_columns(["text"])  # only keep the 'text' column
+>>> wiki = load_dataset("wikimedia/wikipedia", "20231101.en", split="train")
+>>> wiki = wiki.select_columns(["text"])  # only keep the 'text' column
+
+>>> assert stories.features.type == wiki.features.type
+>>> bert_dataset = concatenate_datasets([stories, wiki])
+```
+
+You can also concatenate two datasets horizontally by setting `axis=1` as long as the datasets have the same number of rows:
+
+```py
+>>> from datasets import Dataset
+>>> stories_ids = Dataset.from_dict({"ids": list(range(len(stories)))})
+>>> stories_with_ids = concatenate_datasets([stories, stories_ids], axis=1)
+```
+
+### Interleave
+
+You can also mix several datasets together by taking alternating examples from each one to create a new dataset. This is known as _interleaving_, which is enabled by the [`interleave_datasets`] function. Both [`interleave_datasets`] and [`concatenate_datasets`] work with regular [`Dataset`] and [`IterableDataset`] objects.
+Refer to the [Stream](./stream#interleave) guide for an example of how to interleave [`IterableDataset`] objects.
+
+You can define sampling probabilities for each of the original datasets to specify how to interleave the datasets.
+In this case, the new dataset is constructed by getting examples one by one from a random dataset until one of the datasets runs out of samples.
+
+```py
+>>> from datasets import Dataset, interleave_datasets
+>>> seed = 42
+>>> probabilities = [0.3, 0.5, 0.2]
+>>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
+>>> d2 = Dataset.from_dict({"a": [10, 11, 12, 13]})
+>>> d3 = Dataset.from_dict({"a": [20, 21, 22]})
+>>> dataset = interleave_datasets([d1, d2, d3], probabilities=probabilities, seed=seed)
+>>> dataset["a"]
+[10, 11, 20, 12, 0, 21, 13]
+```
+
+You can also specify the `stopping_strategy`. The default strategy, `first_exhausted`, is a subsampling strategy, i.e the dataset construction is stopped as soon one of the dataset runs out of samples.
+You can specify `stopping_strategy=all_exhausted` to execute an oversampling strategy. In this case, the dataset construction is stopped as soon as every samples in every dataset has been added at least once. In practice, it means that if a dataset is exhausted, it will return to the beginning of this dataset until the stop criterion has been reached.
+Note that if no sampling probabilities are specified, the new dataset will have `max_length_datasets*nb_dataset samples`.
+There is also `stopping_strategy=all_exhausted_without_replacement` to ensure that every sample is seen exactly once.
+
+```py
+>>> d1 = Dataset.from_dict({"a": [0, 1, 2]})
+>>> d2 = Dataset.from_dict({"a": [10, 11, 12, 13]})
+>>> d3 = Dataset.from_dict({"a": [20, 21, 22]})
+>>> dataset = interleave_datasets([d1, d2, d3], stopping_strategy="all_exhausted")
+>>> dataset["a"]
+[0, 10, 20, 1, 11, 21, 2, 12, 22, 0, 13, 20]
+```
+
+## Format
+
+The [`~Dataset.with_format`] function changes the format of a column to be compatible with some common data formats. Specify the output you'd like in the `type` parameter. You can also choose which the columns you want to format using `columns=`. Formatting is applied on-the-fly.
+
+For example, create PyTorch tensors by setting `type="torch"`:
+
+```py
+>>> dataset = dataset.with_format(type="torch")
+```
+
+The [`~Dataset.set_format`] function also changes the format of a column, except it runs in-place:
+
+```py
+>>> dataset.set_format(type="torch")
+```
+
+If you need to reset the dataset to its original format, set the format to `None` (or use [`~Dataset.reset_format`]):
+
+```py
+>>> dataset.format
+{'type': 'torch', 'format_kwargs': {}, 'columns': [...], 'output_all_columns': False}
+>>> dataset = dataset.with_format(None)
+>>> dataset.format
+{'type': None, 'format_kwargs': {}, 'columns': [...], 'output_all_columns': False}
+```
+
+### Tensors formats
+
+Several tensors or arrays formats are supported. It is generally recommended to use these formats instead of converting outputs of a dataset to tensors or arrays manually to avoid unnecessary data copies and accelerate data loading.
+
+Here is the list of supported tensors or arrays formats:
+
+- NumPy: format name is "numpy", for more information see [Using Datasets with NumPy](use_with_numpy)
+- PyTorch: format name is "torch", for more information see [Using Datasets with PyTorch](use_with_pytorch)
+- TensorFlow: format name is "tensorflow", for more information see [Using Datasets with TensorFlow](use_with_tensorflow)
+- JAX: format name is "jax", for more information see [Using Datasets with JAX](use_with_jax)
+
+> [!TIP]
+> Check out the [Using Datasets with TensorFlow](use_with_tensorflow#using-totfdataset) guide for more details on how to efficiently create a TensorFlow dataset.
+
+When a dataset is formatted in a tensor or array format, all the data are formatted as tensors or arrays (except unsupported types like strings for example for PyTorch):
+
+```python
+>>> ds = Dataset.from_dict({"text": ["foo", "bar"], "tokens": [[0, 1, 2], [3, 4, 5]]})
+>>> ds = ds.with_format("torch")
+>>> ds[0]
+{'text': 'foo', 'tokens': tensor([0, 1, 2])}
+>>> ds[:2]
+{'text': ['foo', 'bar'],
+ 'tokens': tensor([[0, 1, 2],
+         [3, 4, 5]])}
+```
+
+### Tabular formats
+
+You can use a dataframes or tables format to optimize data loading and data processing, since they generally offer zero-copy operations and transforms written in low-level languages.
+
+Here is the list of supported dataframes or tables formats:
+
+- Pandas: format name is "pandas", for more information see [Using Datasets with Pandas](use_with_pandas)
+- Polars: format name is "polars", for more information see [Using Datasets with Polars](use_with_polars)
+- PyArrow: format name is "arrow", for more information see [Using Datasets with PyArrow](use_with_tensorflow)
+
+When a dataset is formatted in a dataframe or table format, every dataset row or batches of rows is formatted as a dataframe or table, and dataset colums are formatted as a series or array:
+
+```python
+>>> ds = Dataset.from_dict({"text": ["foo", "bar"], "label": [0, 1]})
+>>> ds = ds.with_format("pandas")
+>>> ds[:2]
+  text  label
+0  foo      0
+1  bar      1
+```
+
+Those formats make it possible to iterate on the data faster by avoiding data copies, and also enable faster data processing in [`~Dataset.map`] or [`~Dataset.filter`]:
+
+```python
+>>> ds = ds.map(lambda df: df.assign(upper_text=df.text.str.upper()), batched=True)
+>>> ds[:2]
+  text  label upper_text
+0  foo      0        FOO
+1  bar      1        BAR
+```
+
+### Custom format transform
+
+The [`~Dataset.with_transform`] function applies a custom formatting transform on-the-fly. This function replaces any previously specified format. For example, you can use this function to tokenize and pad tokens on-the-fly. Tokenization is only applied when examples are accessed:
+
+```py
+>>> from transformers import AutoTokenizer
+
+>>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+>>> def encode(batch):
+...     return tokenizer(batch["sentence1"], batch["sentence2"], padding="longest", truncation=True, max_length=512, return_tensors="pt")
+>>> dataset = dataset.with_transform(encode)
+>>> dataset.format
+{'type': 'custom', 'format_kwargs': {'transform': <function __main__.encode(batch)>}, 'columns': ['idx', 'label', 'sentence1', 'sentence2'], 'output_all_columns': False}
+```
+
+There is also [`~Dataset.set_transform`] which does the same but runs in-place.
+
+You can also use the [`~Dataset.with_transform`] function for custom decoding on [`Features`].
+
+The example below uses the [`pydub`](http://pydub.com/) package as an alternative to `torchcodec` decoding:
+
+```py
+>>> import numpy as np
+>>> from pydub import AudioSegment
+
+>>> audio_dataset_amr = Dataset.from_dict({"audio": ["audio_samples/audio.amr"]})
+
+>>> def decode_audio_with_pydub(batch, sampling_rate=16_000):
+...     def pydub_decode_file(audio_path):
+...         sound = AudioSegment.from_file(audio_path)
+...         if sound.frame_rate != sampling_rate:
+...             sound = sound.set_frame_rate(sampling_rate)
+...         channel_sounds = sound.split_to_mono()
+...         samples = [s.get_array_of_samples() for s in channel_sounds]
+...         fp_arr = np.array(samples).T.astype(np.float32)
+...         fp_arr /= np.iinfo(samples[0].typecode).max
+...         return fp_arr
+...
+...     batch["audio"] = [pydub_decode_file(audio_path) for audio_path in batch["audio"]]
+...     return batch
+
+>>> audio_dataset_amr.set_transform(decode_audio_with_pydub)
+```
+
+## Save
+
+Once your dataset is ready, you can save it as a Hugging Face Dataset in Parquet format and reuse it later with [`load_dataset`].
+
+Save your dataset by providing the name of the dataset repository on Hugging Face you wish to save it to to [`~Dataset.push_to_hub`]:
+
+```python
+encoded_dataset.push_to_hub("username/my_dataset")
+```
+
+You can use multiple processes to upload it in parallel. This is especially useful if you want to speed up the process:
+
+```python
+dataset.push_to_hub("username/my_dataset", num_proc=8)
+```
+
+Use the [`load_dataset`] function to reload the dataset (in streaming mode or not):
+
+```python
+from datasets import load_dataset
+reloaded_dataset = load_dataset("username/my_dataset", streaming=True)
+```
+
+Alternatively, you can save it locally in Arrow format on disk. Compared to Parquet, Arrow is uncompressed which makes it much faster to reload which is great for local use on disk and ephemeral caching. But since it's larger and with less metadata, it is slower to upload/download/query than Parquet and less suited for long term storage.
+
+Use the [`~Dataset.save_to_disk`] and [`load_from_disk`] function to reload the dataset from your disk:
+
+```py
+>>> encoded_dataset.save_to_disk("path/of/my/dataset/directory")
+>>> # later
+>>> from datasets import load_from_disk
+>>> reloaded_dataset = load_from_disk("path/of/my/dataset/directory")
+```
+
+## Export
+
+🤗 Datasets supports exporting as well so you can work with your dataset in other applications. The following table shows currently supported file formats you can export to:
+
+| File type               | Export method                                                       |
+| ----------------------- | ------------------------------------------------------------------- |
+| CSV                     | [`Dataset.to_csv`]                                                  |
+| JSON                    | [`Dataset.to_json`]                                                 |
+| Parquet                 | [`Dataset.to_parquet`]                                              |
+| SQL                     | [`Dataset.to_sql`]                                                  |
+| In-memory Python object | [`Dataset.to_pandas`], [`Dataset.to_polars`] or [`Dataset.to_dict`] |
+
+For example, export your dataset to a CSV file like this:
+
+```py
+>>> encoded_dataset.to_csv("path/of/my/dataset.csv")
+```
+
+Use a `hf://` path to export to a [Dataset repository](https://huggingface.co/docs/hub/datasets-overview) or a [Storage Bucket](https://huggingface.co/docs/hub/storage-buckets) on Hugging Face:
+
+```py
+>>> encoded_dataset.to_csv("hf://datasets/username/dataset_name/path/of/my/dataset.csv")
+>>> encoded_dataset.to_csv("hf://buckets/username/raw_data_bucket/path/of/my/dataset.csv")
+```

docs/source/quickstart.mdx

@@ -0,0 +1,380 @@
+<!--Copyright 2023 The HuggingFace Team. All rights reserved.
+
+Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with
+the License. You may obtain a copy of the License at
+
+http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software distributed under the License is distributed on
+an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License for the
+specific language governing permissions and limitations under the License.
+-->
+
+# Quickstart
+
+[[open-in-colab]]
+
+This quickstart is intended for developers who are ready to dive into the code and see an example of how to integrate 🤗 Datasets into their model training workflow. If you're a beginner, we recommend starting with our [tutorials](./tutorial), where you'll get a more thorough introduction.
+
+Each dataset is unique, and depending on the task, some datasets may require additional steps to prepare it for training. But you can always use 🤗 Datasets tools to load and process a dataset. The fastest and easiest way to get started is by loading an existing dataset from the [Hugging Face Hub](https://huggingface.co/datasets). There are thousands of datasets to choose from, spanning many tasks. Choose the type of dataset you want to work with, and let's get started!
+
+<div class="mt-4">
+  <div class="w-full flex flex-col space-y-4 md:space-y-0 md:grid md:grid-cols-3 md:gap-y-4 md:gap-x-5">
+    <a
+      class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg"
+      href="#audio"
+    >
+      <div class="w-full text-center bg-gradient-to-r from-violet-300 via-sky-400 to-green-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">
+        Audio
+      </div>
+      <p class="text-gray-700">
+        Resample an audio dataset and get it ready for a model to classify what
+        type of banking issue a speaker is calling about.
+      </p>
+    </a>
+    <a
+      class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg"
+      href="#vision"
+    >
+      <div class="w-full text-center bg-gradient-to-r from-pink-400 via-purple-400 to-blue-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">
+        Vision
+      </div>
+      <p class="text-gray-700">
+        Apply data augmentation to an image dataset and get it ready for a model
+        to diagnose disease in bean plants.
+      </p>
+    </a>
+    <a
+      class="!no-underline border dark:border-gray-700 p-5 rounded-lg shadow hover:shadow-lg"
+      href="#nlp"
+    >
+      <div class="w-full text-center bg-gradient-to-r from-orange-300 via-red-400 to-violet-500 rounded-lg py-1.5 font-semibold mb-5 text-white text-lg leading-relaxed">
+        NLP
+      </div>
+      <p class="text-gray-700">
+        Tokenize a dataset and get it ready for a model to determine whether a
+        pair of sentences have the same meaning.
+      </p>
+    </a>
+  </div>
+</div>
+
+> [!TIP]
+> Check out [Chapter 5](https://huggingface.co/course/chapter5/1?fw=pt) of the Hugging Face course to learn more about other important topics such as loading remote or local datasets, tools for cleaning up a dataset, and creating your own dataset.
+
+Start by installing 🤗 Datasets:
+
+```bash
+pip install datasets
+```
+
+🤗 Datasets also support audio and image data formats:
+
+- To work with audio datasets, install the [`Audio`] feature:
+
+  ```bash
+  pip install datasets[audio]
+  ```
+
+- To work with image datasets, install the [`Image`] feature:
+
+  ```bash
+  pip install datasets[vision]
+  ```
+
+Besides 🤗 Datasets, make sure your preferred machine learning framework is installed:
+
+<frameworkcontent>
+  <pt>```bash pip install torch ```</pt>
+  <tf>```bash pip install tensorflow ```</tf>
+</frameworkcontent>
+
+## Audio
+
+Audio datasets are loaded just like text datasets. However, an audio dataset is preprocessed a bit differently. Instead of a tokenizer, you'll need a [feature extractor](https://huggingface.co/docs/transformers/main_classes/feature_extractor#feature-extractor). An audio input may also require resampling its sampling rate to match the sampling rate of the pretrained model you're using. In this quickstart, you'll prepare the [MInDS-14](https://huggingface.co/datasets/PolyAI/minds14) dataset for a model train on and classify the banking issue a customer is having.
+
+**1**. Load the MInDS-14 dataset by providing the [`load_dataset`] function with the dataset name, dataset configuration (not all datasets will have a configuration), and a dataset split:
+
+```py
+>>> from datasets import load_dataset, Audio
+
+>>> dataset = load_dataset("PolyAI/minds14", "en-US", split="train")
+```
+
+**2**. Next, load a pretrained [Wav2Vec2](https://huggingface.co/facebook/wav2vec2-base) model and its corresponding feature extractor from the [🤗 Transformers](https://huggingface.co/transformers/) library. It is totally normal to see a warning after you load the model about some weights not being initialized. This is expected because you are loading this model checkpoint for training with another task.
+
+```py
+>>> from transformers import AutoModelForAudioClassification, AutoFeatureExtractor
+
+>>> model = AutoModelForAudioClassification.from_pretrained("facebook/wav2vec2-base")
+>>> feature_extractor = AutoFeatureExtractor.from_pretrained("facebook/wav2vec2-base")
+```
+
+**3**. The [MInDS-14](https://huggingface.co/datasets/PolyAI/minds14) dataset card indicates the sampling rate is 8kHz, but the Wav2Vec2 model was pretrained on a sampling rate of 16kHZ. You'll need to upsample the `audio` column with the [`~Dataset.cast_column`] function and [`Audio`] feature to match the model's sampling rate.
+
+```py
+>>> dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
+>>> dataset[0]["audio"]
+<datasets.features._torchcodec.AudioDecoder object at 0x11642b6a0>
+```
+
+**4**. Create a function to preprocess the audio `array` with the feature extractor, and truncate and pad the sequences into tidy rectangular tensors. The most important thing to remember is to call the audio `array` in the feature extractor since the `array` - the actual speech signal - is the model input.
+
+Once you have a preprocessing function, use the [`~Dataset.map`] function to speed up processing by applying the function to batches of examples in the dataset.
+
+```py
+>>> def preprocess_function(examples):
+...     audio_arrays = [x.get_all_samples().data for x in examples["audio"]]
+...     inputs = feature_extractor(
+...         audio_arrays,
+...         sampling_rate=16000,
+...         padding=True,
+...         max_length=100000,
+...         truncation=True,
+...     )
+...     return inputs
+
+>>> dataset = dataset.map(preprocess_function, batched=True)
+```
+
+**5**. Use the [`~Dataset.rename_column`] function to rename the `intent_class` column to `labels`, which is the expected input name in [Wav2Vec2ForSequenceClassification](https://huggingface.co/docs/transformers/main/en/model_doc/wav2vec2#transformers.Wav2Vec2ForSequenceClassification):
+
+```py
+>>> dataset = dataset.rename_column("intent_class", "labels")
+```
+
+**6**. Set the dataset format according to the machine learning framework you're using.
+
+<frameworkcontent>
+<pt>
+Use the [`~Dataset.set_format`] function to set the dataset format to `torch` and specify the columns you want to format. This function applies formatting on-the-fly. After converting to PyTorch tensors, wrap the dataset in [`torch.utils.data.DataLoader`](https://alband.github.io/doc_view/data.html?highlight=torch%20utils%20data%20dataloader#torch.utils.data.DataLoader):
+
+```py
+>>> from torch.utils.data import DataLoader
+
+>>> dataset.set_format(type="torch", columns=["input_values", "labels"])
+>>> dataloader = DataLoader(dataset, batch_size=4)
+```
+
+</pt>
+<tf>
+
+Use the [`~transformers.TFPreTrainedModel.prepare_tf_dataset`] method from 🤗 Transformers to prepare the dataset to be compatible with
+TensorFlow, and ready to train/fine-tune a model, as it wraps a HuggingFace [`~datasets.Dataset`] as a `tf.data.Dataset`
+with collation and batching, so one can pass it directly to Keras methods like `fit()` without further modification.
+
+```py
+>>> import tensorflow as tf
+
+>>> tf_dataset = model.prepare_tf_dataset(
+...     dataset,
+...     batch_size=4,
+...     shuffle=True,
+... )
+```
+
+</tf>
+</frameworkcontent>
+
+**7**. Start training with your machine learning framework! Check out the 🤗 Transformers [audio classification guide](https://huggingface.co/docs/transformers/tasks/audio_classification) for an end-to-end example of how to train a model on an audio dataset.
+
+## Vision
+
+Image datasets are loaded just like text datasets. However, instead of a tokenizer, you'll need a [feature extractor](https://huggingface.co/docs/transformers/main_classes/feature_extractor#feature-extractor) to preprocess the dataset. Applying data augmentation to an image is common in computer vision to make the model more robust against overfitting. You're free to use any data augmentation library you want, and then you can apply the augmentations with 🤗 Datasets. In this quickstart, you'll load the [Beans](https://huggingface.co/datasets/beans) dataset and get it ready for the model to train on and identify disease from the leaf images.
+
+**1**. Load the Beans dataset by providing the [`load_dataset`] function with the dataset name and a dataset split:
+
+```py
+>>> from datasets import load_dataset, Image
+
+>>> dataset = load_dataset("AI-Lab-Makerere/beans", split="train")
+```
+
+Most image models work with RBG images. If your dataset contains images in a different mode, you can use the [`~Dataset.cast_column`] function to set the mode to RGB:
+
+```py
+>>> dataset = dataset.cast_column("image", Image(mode="RGB"))
+```
+
+The Beans dataset contains only RGB images, so this step is unnecessary here.
+
+**2**. Now you can add some data augmentations with any library ([Albumentations](https://albumentations.ai/), [imgaug](https://imgaug.readthedocs.io/en/latest/), [Kornia](https://kornia.readthedocs.io/en/latest/)) you like. Here, you'll use [torchvision](https://pytorch.org/vision/stable/transforms.html) to randomly change the color properties of an image:
+
+```py
+>>> from torchvision.transforms import Compose, ColorJitter, ToTensor
+
+>>> jitter = Compose(
+...     [ColorJitter(brightness=0.5, hue=0.5), ToTensor()]
+... )
+```
+
+**3**. Create a function to apply your transform to the dataset and generate the model input: `pixel_values`.
+
+```python
+>>> def transforms(examples):
+...     examples["pixel_values"] = [jitter(image.convert("RGB")) for image in examples["image"]]
+...     return examples
+```
+
+**4**. Use the [`~Dataset.with_transform`] function to apply the data augmentations on-the-fly:
+
+```py
+>>> dataset = dataset.with_transform(transforms)
+```
+
+**5**. Set the dataset format according to the machine learning framework you're using.
+
+<frameworkcontent>
+<pt>
+Wrap the dataset in [`torch.utils.data.DataLoader`](https://alband.github.io/doc_view/data.html?highlight=torch%20utils%20data%20dataloader#torch.utils.data.DataLoader). You'll also need to create a collate function to collate the samples into batches:
+
+```py
+>>> from torch.utils.data import DataLoader
+
+>>> def collate_fn(examples):
+...     images = []
+...     labels = []
+...     for example in examples:
+...         images.append((example["pixel_values"]))
+...         labels.append(example["labels"])
+...
+...     pixel_values = torch.stack(images)
+...     labels = torch.tensor(labels)
+...     return {"pixel_values": pixel_values, "labels": labels}
+>>> dataloader = DataLoader(dataset, collate_fn=collate_fn, batch_size=4)
+```
+
+</pt>
+<tf>
+
+Use the [`~transformers.TFPreTrainedModel.prepare_tf_dataset`] method from 🤗 Transformers to prepare the dataset to be compatible with
+TensorFlow, and ready to train/fine-tune a model, as it wraps a HuggingFace [`~datasets.Dataset`] as a `tf.data.Dataset`
+with collation and batching, so one can pass it directly to Keras methods like `fit()` without further modification.
+
+Before you start, make sure you have up-to-date versions of `albumentations` and `cv2` installed:
+
+```bash
+pip install -U albumentations opencv-python
+```
+
+```py
+>>> import albumentations
+>>> import numpy as np
+
+>>> transform = albumentations.Compose([
+...     albumentations.RandomCrop(width=256, height=256),
+...     albumentations.HorizontalFlip(p=0.5),
+...     albumentations.RandomBrightnessContrast(p=0.2),
+... ])
+
+>>> def transforms(examples):
+...     examples["pixel_values"] = [
+...         transform(image=np.array(image))["image"] for image in examples["image"]
+...     ]
+...     return examples
+
+>>> dataset.set_transform(transforms)
+>>> tf_dataset = model.prepare_tf_dataset(
+...     dataset,
+...     batch_size=4,
+...     shuffle=True,
+... )
+```
+
+</tf>
+</frameworkcontent>
+
+**6**. Start training with your machine learning framework! Check out the 🤗 Transformers [image classification guide](https://huggingface.co/docs/transformers/tasks/image_classification) for an end-to-end example of how to train a model on an image dataset.
+
+## NLP
+
+Text needs to be tokenized into individual tokens by a [tokenizer](https://huggingface.co/docs/transformers/main_classes/tokenizer). For the quickstart, you'll load the [Microsoft Research Paraphrase Corpus (MRPC)](https://huggingface.co/datasets/nyu-mll/glue/viewer/mrpc) training dataset to train a model to determine whether a pair of sentences mean the same thing.
+
+**1**. Load the MRPC dataset by providing the [`load_dataset`] function with the dataset name, dataset configuration (not all datasets will have a configuration), and dataset split:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("nyu-mll/glue", "mrpc", split="train")
+```
+
+**2**. Next, load a pretrained [BERT](https://huggingface.co/bert-base-uncased) model and its corresponding tokenizer from the [🤗 Transformers](https://huggingface.co/transformers/) library. It is totally normal to see a warning after you load the model about some weights not being initialized. This is expected because you are loading this model checkpoint for training with another task.
+
+```py
+>>> from transformers import AutoModelForSequenceClassification, AutoTokenizer
+
+>>> model = AutoModelForSequenceClassification.from_pretrained("bert-base-uncased")
+>>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+===PT-TF-SPLIT===
+>>> from transformers import TFAutoModelForSequenceClassification, AutoTokenizer
+
+>>> model = TFAutoModelForSequenceClassification.from_pretrained("bert-base-uncased")
+>>> tokenizer = AutoTokenizer.from_pretrained("bert-base-uncased")
+```
+
+**3**. Create a function to tokenize the dataset, and you should also truncate and pad the text into tidy rectangular tensors. The tokenizer generates three new columns in the dataset: `input_ids`, `token_type_ids`, and an `attention_mask`. These are the model inputs.
+
+Use the [`~Dataset.map`] function to speed up processing by applying your tokenization function to batches of examples in the dataset:
+
+```py
+>>> def encode(examples):
+...     return tokenizer(examples["sentence1"], examples["sentence2"], truncation=True, padding="max_length")
+
+>>> dataset = dataset.map(encode, batched=True)
+>>> dataset[0]
+{'sentence1': 'Amrozi accused his brother , whom he called " the witness " , of deliberately distorting his evidence .',
+'sentence2': 'Referring to him as only " the witness " , Amrozi accused his brother of deliberately distorting his evidence .',
+'label': 1,
+'idx': 0,
+'input_ids': [  101,  7277,  2180,  5303,  4806,  1117,  1711,   117,  2292, 1119,  1270,   107,  1103,  7737,   107,   117,  1104,  9938, 4267, 12223, 21811,  1117,  2554,   119,   102, 11336,  6732, 3384,  1106,  1140,  1112,  1178,   107,  1103,  7737,   107, 117,  7277,  2180,  5303,  4806,  1117,  1711,  1104,  9938, 4267, 12223, 21811,  1117,  2554,   119,   102, 0, 0, ...],
+'token_type_ids': [0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 0, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, ...],
+'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 0, 0, ...]}
+```
+
+**4**. Rename the `label` column to `labels`, which is the expected input name in [BertForSequenceClassification](https://huggingface.co/docs/transformers/main/en/model_doc/bert#transformers.BertForSequenceClassification):
+
+```py
+>>> dataset = dataset.map(lambda examples: {"labels": examples["label"]}, batched=True)
+```
+
+**5**. Set the dataset format according to the machine learning framework you're using.
+
+<frameworkcontent>
+<pt>
+Use the [`~Dataset.with_format`] function to set the dataset format to `torch` and specify the columns you want to format. This function applies formatting on-the-fly. After converting to PyTorch tensors, wrap the dataset in [`torch.utils.data.DataLoader`](https://alband.github.io/doc_view/data.html?highlight=torch%20utils%20data%20dataloader#torch.utils.data.DataLoader):
+
+```py
+>>> import torch
+
+>>> dataset = dataset.select_columns(["input_ids", "token_type_ids", "attention_mask", "labels"])
+>>> dataset = dataset.with_format(type="torch")
+>>> dataloader = torch.utils.data.DataLoader(dataset, batch_size=32)
+```
+
+</pt>
+<tf>
+
+Use the [`~transformers.TFPreTrainedModel.prepare_tf_dataset`] method from 🤗 Transformers to prepare the dataset to be compatible with
+TensorFlow, and ready to train/fine-tune a model, as it wraps a HuggingFace [`~datasets.Dataset`] as a `tf.data.Dataset`
+with collation and batching, so one can pass it directly to Keras methods like `fit()` without further modification.
+
+```py
+>>> import tensorflow as tf
+
+>>> tf_dataset = model.prepare_tf_dataset(
+...     dataset,
+...     batch_size=4,
+...     shuffle=True,
+... )
+```
+
+</tf>
+</frameworkcontent>
+
+**6**. Start training with your machine learning framework! Check out the 🤗 Transformers [text classification guide](https://huggingface.co/docs/transformers/tasks/sequence_classification) for an end-to-end example of how to train a model on a text dataset.
+
+## What's next?
+
+This completes the 🤗 Datasets quickstart! You can load any text, audio, or image dataset with a single function and get it ready for your model to train on.
+
+For your next steps, take a look at our [How-to guides](./how_to) and learn how to do more specific things like loading different dataset formats, aligning labels, and streaming large datasets. If you're interested in learning more about 🤗 Datasets core concepts, grab a cup of coffee and read our [Conceptual Guides](./about_arrow)!

docs/source/repository_structure.mdx

@@ -0,0 +1,273 @@
+# Structure your repository
+
+To host and share your dataset, create a dataset repository on the Hugging Face Hub and upload your data files.
+
+This guide will show you how to structure your dataset repository when you upload it.
+A dataset with a supported structure and file format (`.txt`, `.csv`, `.parquet`, `.jsonl`, `.mp3`, `.jpg`, `.zip` etc.) are loaded automatically with [`~datasets.load_dataset`], and it'll have a dataset viewer on its dataset page on the Hub.
+
+## Main use-case
+
+The simplest dataset structure has two files: `train.csv` and `test.csv` (this works with any supported file format).
+
+Your repository will also contain a `README.md` file, the [dataset card](dataset_card) displayed on your dataset page.
+
+```
+my_dataset_repository/
+├── README.md
+├── train.csv
+└── test.csv
+```
+
+In this simple case, you'll get a dataset with two splits: `train` (containing examples from `train.csv`) and `test` (containing examples from `test.csv`).
+
+## Define your splits and subsets in YAML
+
+## Splits
+
+If you have multiple files and want to define which file goes into which split, you can use the YAML `configs` field at the top of your README.md.
+
+For example, given a repository like this one:
+
+```
+my_dataset_repository/
+├── README.md
+├── data.csv
+└── holdout.csv
+```
+
+You can define your splits by adding the `configs` field in the YAML block at the top of your README.md:
+
+```yaml
+---
+configs:
+- config_name: default
+  data_files:
+  - split: train
+    path: "data.csv"
+  - split: test
+    path: "holdout.csv"
+---
+```
+
+
+You can select multiple files per split using a list of paths:
+
+```
+my_dataset_repository/
+├── README.md
+├── data/
+│   ├── abc.csv
+│   └── def.csv
+└── holdout/
+    └── ghi.csv
+```
+
+```yaml
+---
+configs:
+- config_name: default
+  data_files:
+  - split: train
+    path:
+    - "data/abc.csv"
+    - "data/def.csv"
+  - split: test
+    path: "holdout/ghi.csv"
+---
+```
+
+Or you can use glob patterns to automatically list all the files you need:
+
+```yaml
+---
+configs:
+- config_name: default
+  data_files:
+  - split: train
+    path: "data/*.csv"
+  - split: test
+    path: "holdout/*.csv"
+---
+```
+
+> [!WARNING]
+> Note that `config_name` field is required even if you have a single configuration.
+
+## Configurations
+
+Your dataset might have several subsets of data that you want to be able to load separately. In that case you can define a list of configurations inside the `configs` field in YAML:
+
+```
+my_dataset_repository/
+├── README.md
+├── main_data.csv
+└── additional_data.csv
+```
+
+```yaml
+---
+configs:
+- config_name: main_data
+  data_files: "main_data.csv"
+- config_name: additional_data
+  data_files: "additional_data.csv"
+---
+```
+
+Each configuration is shown separately on the Hugging Face Hub, and can be loaded by passing its name as a second parameter:
+
+```python
+from datasets import load_dataset
+
+main_data = load_dataset("my_dataset_repository", "main_data")
+additional_data = load_dataset("my_dataset_repository", "additional_data")
+```
+
+## Builder parameters
+
+Not only `data_files`, but other builder-specific parameters can be passed via YAML, allowing for more flexibility on how to load the data while not requiring any custom code. For example, define which separator to use in which configuration to load your `csv` files:
+
+```yaml
+---
+configs:
+- config_name: tab
+  data_files: "main_data.csv"
+  sep: "\t"
+- config_name: comma
+  data_files: "additional_data.csv"
+  sep: ","
+---
+```
+
+Refer to [specific builders' documentation](./package_reference/builder_classes) to see what configuration parameters they have.
+
+> [!TIP]
+> You can set a default configuration using `default: true`, e.g. you can run `main_data = load_dataset("my_dataset_repository")` if you set 
+>
+> ```yaml
+> - config_name: main_data
+>   data_files: "main_data.csv"
+>   default: true
+> ```
+
+## Automatic splits detection
+
+If no YAML is provided, 🤗 Datasets searches for certain patterns in the dataset repository to automatically infer the dataset splits.
+There is an order to the patterns, beginning with the custom filename split format to treating all files as a single split if no pattern is found.
+
+### Directory name
+
+Your data files may also be placed into different directories named `train`, `test`, and `validation` where each directory contains the data files for that split:
+
+```
+my_dataset_repository/
+├── README.md
+└── data/
+    ├── train/
+    │   └── bees.csv
+    ├── test/
+    │   └── more_bees.csv
+    └── validation/
+        └── even_more_bees.csv
+```
+
+### Filename splits
+
+If you don't have any non-traditional splits, then you can place the split name anywhere in the data file and it is automatically inferred. The only rule is that the split name must be delimited by non-word characters, like `test-file.csv` for example instead of `testfile.csv`. Supported delimiters include underscores, dashes, spaces, dots, and numbers.
+
+For example, the following file names are all acceptable:
+
+- train split: `train.csv`, `my_train_file.csv`, `train1.csv`
+- validation split: `validation.csv`, `my_validation_file.csv`, `validation1.csv`
+- test split: `test.csv`, `my_test_file.csv`, `test1.csv`
+
+Here is an example where all the files are placed into a directory named `data`:
+
+```
+my_dataset_repository/
+├── README.md
+└── data/
+    ├── train.csv
+    ├── test.csv
+    └── validation.csv
+```
+
+### Custom filename split
+
+If your dataset splits have custom names that aren't `train`, `test`, or `validation`, then you can name your data files like `data/<split_name>-xxxxx-of-xxxxx.csv`.
+
+Here is an example with three splits, `train`, `test`, and `random`:
+
+```
+my_dataset_repository/
+├── README.md
+└── data/
+    ├── train-00000-of-00003.csv
+    ├── train-00001-of-00003.csv
+    ├── train-00002-of-00003.csv
+    ├── test-00000-of-00001.csv
+    ├── random-00000-of-00003.csv
+    ├── random-00001-of-00003.csv
+    └── random-00002-of-00003.csv
+```
+
+### Single split
+
+When 🤗 Datasets can't find any of the above patterns, then it'll treat all the files as a single train split. If your dataset splits aren't loading as expected, it may be due to an incorrect pattern.
+
+### Split name keywords
+
+There are several ways to name splits. Validation splits are sometimes called "dev", and test splits may be referred to as "eval".
+These other split names are also supported, and the following keywords are equivalent:
+
+- train, training
+- validation, valid, val, dev
+- test, testing, eval, evaluation
+
+The structure below is a valid repository:
+
+```
+my_dataset_repository/
+├── README.md
+└── data/
+    ├── training.csv
+    ├── eval.csv
+    └── valid.csv
+```
+
+### Multiple files per split
+
+If one of your splits comprises several files, 🤗 Datasets can still infer whether it is the train, validation, and test split from the file name.
+For example, if your train and test splits span several files:
+
+```
+my_dataset_repository/
+├── README.md
+├── train_0.csv
+├── train_1.csv
+├── train_2.csv
+├── train_3.csv
+├── test_0.csv
+└── test_1.csv
+```
+
+Make sure all the files of your `train` set have *train* in their names (same for test and validation).
+Even if you add a prefix or suffix to `train` in the file name (like `my_train_file_00001.csv` for example),
+🤗 Datasets can still infer the appropriate split.
+
+For convenience, you can also place your data files into different directories.
+In this case, the split name is inferred from the directory name.
+
+```
+my_dataset_repository/
+├── README.md
+└── data/
+    ├── train/
+    │   ├── shard_0.csv
+    │   ├── shard_1.csv
+    │   ├── shard_2.csv
+    │   └── shard_3.csv
+    └── test/
+        ├── shard_0.csv
+        └── shard_1.csv
+```

docs/source/semantic_segmentation.mdx

@@ -0,0 +1,174 @@
+# Semantic segmentation
+
+Semantic segmentation datasets are used to train a model to classify every pixel in an image. There are
+a wide variety of applications enabled by these datasets such as background removal from images, stylizing
+images, or scene understanding for autonomous driving. This guide will show you how to apply transformations
+to an image segmentation dataset.
+
+Before you start, make sure you have up-to-date versions of `albumentations` and `cv2` installed:
+
+```bash
+pip install -U albumentations opencv-python
+```
+
+[Albumentations](https://albumentations.ai/) is a Python library for performing data augmentation
+for computer vision. It supports various computer vision tasks such as image classification, object
+detection, segmentation, and keypoint estimation.
+
+This guide uses the [Scene Parsing](https://huggingface.co/datasets/scene_parse_150) dataset for segmenting
+and parsing an image into different image regions associated with semantic categories, such as sky, road, person, and bed.
+
+Load the `train` split of the dataset and take a look at an example:
+
+```py
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("scene_parse_150", split="train")
+>>> index = 10
+>>> dataset[index]
+{'image': <PIL.JpegImagePlugin.JpegImageFile image mode=RGB size=683x512 at 0x7FB37B0EC810>,
+ 'annotation': <PIL.PngImagePlugin.PngImageFile image mode=L size=683x512 at 0x7FB37B0EC9D0>,
+ 'scene_category': 927}
+```
+
+The dataset has three fields:
+
+* `image`: a PIL image object.
+* `annotation`: segmentation mask of the image.
+* `scene_category`: the label or scene category of the image (like “kitchen” or “office”).
+
+Next, check out an image with:
+
+```py
+>>> dataset[index]["image"]
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/image_seg.png">
+</div>
+
+Similarly, you can check out the respective segmentation mask:
+
+```py
+>>> dataset[index]["annotation"]
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/seg_mask.png">
+</div>
+
+We can also add a [color palette](https://github.com/tensorflow/models/blob/3f1ca33afe3c1631b733ea7e40c294273b9e406d/research/deeplab/utils/get_dataset_colormap.py#L51) on the
+segmentation mask and overlay it on top of the original image to visualize the dataset:
+
+After defining the color palette, you should be ready to visualize some overlays. 
+
+```py
+>>> import matplotlib.pyplot as plt
+
+>>> def visualize_seg_mask(image: np.ndarray, mask: np.ndarray):
+...    color_seg = np.zeros((mask.shape[0], mask.shape[1], 3), dtype=np.uint8)
+...    palette = np.array(create_ade20k_label_colormap())
+...    for label, color in enumerate(palette):
+...        color_seg[mask == label, :] = color
+...    color_seg = color_seg[..., ::-1]  # convert to BGR
+
+...    img = np.array(image) * 0.5 + color_seg * 0.5  # plot the image with the segmentation map
+...    img = img.astype(np.uint8)
+
+...    plt.figure(figsize=(15, 10))
+...    plt.imshow(img)
+...    plt.axis("off")
+...    plt.show()
+
+
+>>> visualize_seg_mask(
+...     np.array(dataset[index]["image"]),
+...     np.array(dataset[index]["annotation"])
+... )
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/seg_overlay.png">
+</div>
+
+Now apply some augmentations with `albumentations`. You’ll first resize the image and adjust its brightness.
+
+```py
+>>> import albumentations
+
+>>> transform = albumentations.Compose(
+...     [
+...         albumentations.Resize(256, 256),
+...         albumentations.RandomBrightnessContrast(brightness_limit=0.3, contrast_limit=0.3, p=0.5),
+...     ]
+... )
+```
+
+Create a function to apply the transformation to the images:
+
+```py
+>>> def transforms(examples):
+...     transformed_images, transformed_masks = [], []
+...
+...     for image, seg_mask in zip(examples["image"], examples["annotation"]):
+...         image, seg_mask = np.array(image), np.array(seg_mask)
+...         transformed = transform(image=image, mask=seg_mask)
+...         transformed_images.append(transformed["image"])
+...         transformed_masks.append(transformed["mask"])
+...
+...     examples["pixel_values"] = transformed_images
+...     examples["label"] = transformed_masks
+...     return examples
+```
+
+Use the [`~Dataset.set_transform`] function to apply the transformation on-the-fly to batches of the dataset to consume less disk space:
+
+```py
+>>> dataset.set_transform(transforms)
+```
+
+You can verify the transformation worked by indexing into the `pixel_values` and `label` of an example:
+
+```py
+>>> image = np.array(dataset[index]["pixel_values"])
+>>> mask = np.array(dataset[index]["label"])
+
+>>> visualize_seg_mask(image, mask)
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/albumentations_seg.png">
+</div>
+
+In this guide, you have used `albumentations` for augmenting the dataset. It's also possible to use `torchvision` to apply some similar transforms. 
+
+```py 
+>>> from torchvision.transforms import Resize, ColorJitter, Compose
+
+>>> transformation_chain = Compose([
+...     Resize((256, 256)),
+...     ColorJitter(brightness=0.25, contrast=0.25, saturation=0.25, hue=0.1)
+... ])
+>>> resize = Resize((256, 256))
+
+>>> def train_transforms(example_batch):
+...     example_batch["pixel_values"] = [transformation_chain(x) for x in example_batch["image"]]
+...     example_batch["label"] = [resize(x) for x in example_batch["annotation"]]
+...     return example_batch
+
+>>> dataset.set_transform(train_transforms)
+
+>>> image = np.array(dataset[index]["pixel_values"])
+>>> mask = np.array(dataset[index]["label"])
+
+>>> visualize_seg_mask(image, mask)
+```
+
+<div class="flex justify-center">
+    <img src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/torchvision_seg.png">
+</div>
+
+> [!TIP]
+> Now that you know how to process a dataset for semantic segmentation, learn
+> [how to train a semantic segmentation model](https://huggingface.co/docs/transformers/tasks/semantic_segmentation)
+> and use it for inference.

docs/source/share.mdx

@@ -0,0 +1,211 @@
+# Share a dataset using the CLI
+
+At Hugging Face, we are on a mission to democratize good Machine Learning and we believe in the value of open source. That's why we designed 🤗 Datasets so that anyone can share a dataset with the greater ML community. There are currently thousands of datasets in over 100 languages in the Hugging Face Hub, and the Hugging Face team always welcomes new contributions!
+
+Dataset repositories offer features such as:
+
+- Free dataset hosting
+- Dataset versioning
+- Commit history and diffs
+- Metadata for discoverability
+- Dataset cards for documentation, licensing, limitations, etc.
+- [Dataset Viewer](https://huggingface.co/docs/hub/datasets-viewer)
+
+This guide will show you how to share a dataset folder or repository that can be easily accessed by anyone.
+
+<a id='upload_dataset_repo'></a>
+
+## Add a dataset
+
+You can share your dataset with the community with a dataset repository on the Hugging Face Hub.
+It can also be a private dataset if you want to control who has access to it.
+
+In a dataset repository, you can host all your data files and [configure your dataset](./repository_structure#define-your-splits-in-yaml) to define which file goes to which split.
+The following formats are supported: CSV, TSV, JSON, JSON lines, text, Parquet, Arrow, SQLite, WebDataset.
+Many kinds of compressed file types are also supported: GZ, BZ2, LZ4, LZMA or ZSTD.
+For example, your dataset can be made of `.json.gz` files.
+
+When loading a dataset from the Hub, all the files in the supported formats are loaded, following the [repository structure](./repository_structure).
+
+For more information on how to load a dataset from the Hub, take a look at the [load a dataset from the Hub](./load_hub) tutorial.
+
+### Create the repository
+
+Sharing a community dataset will require you to create an account on [hf.co](https://huggingface.co/join) if you don't have one yet.
+You can directly create a [new dataset repository](https://huggingface.co/login?next=%2Fnew-dataset) from your account on the Hugging Face Hub, but this guide will show you how to upload a dataset from the terminal.
+
+1. Make sure you are in the virtual environment where you installed Datasets, and run the following command:
+
+```
+huggingface-cli login
+```
+
+2. Login using your Hugging Face Hub credentials, and create a new dataset repository:
+
+```
+huggingface-cli repo create my-cool-dataset --type dataset
+```
+
+Add the `-organization` flag to create a repository under a specific organization:
+
+```
+huggingface-cli repo create my-cool-dataset --type dataset --organization your-org-name
+```
+
+## Prepare your files
+
+Check your directory to ensure the only files you're uploading are:
+
+- The data files of the dataset
+
+- The dataset card `README.md`
+
+
+## huggingface-cli upload
+
+Use the `huggingface-cli upload` command to upload files to the Hub directly. Internally, it uses the same [`upload_file`] and [`upload_folder`] helpers described in the [Upload guide](https://huggingface.co/docs/huggingface_hub/guides/upload). In the examples below, we will walk through the most common use cases. For a full list of available options, you can run:
+
+```bash
+>>> huggingface-cli upload --help
+```
+
+For more general information about `huggingface-cli` you can check the [CLI guide](https://huggingface.co/docs/huggingface_hub/guides/cli).
+
+### Upload an entire folder
+
+The default usage for this command is:
+
+```bash
+# Usage:  huggingface-cli upload [dataset_repo_id] [local_path] [path_in_repo] --repo-type dataset
+```
+
+To upload the current directory at the root of the repo, use:
+
+```bash
+>>> huggingface-cli upload my-cool-dataset . . --repo-type dataset
+https://huggingface.co/datasets/Wauplin/my-cool-dataset/tree/main/
+```
+
+> [!TIP]
+> If the repo doesn't exist yet, it will be created automatically.
+
+You can also upload a specific folder:
+
+```bash
+>>> huggingface-cli upload my-cool-dataset ./data . --repo-type dataset
+https://huggingface.co/datasetsWauplin/my-cool-dataset/tree/main/
+```
+
+Finally, you can upload a folder to a specific destination on the repo:
+
+```bash
+>>> huggingface-cli upload my-cool-dataset ./path/to/curated/data /data/train --repo-type dataset
+https://huggingface.co/datasetsWauplin/my-cool-dataset/tree/main/data/train
+```
+
+### Upload a single file
+
+You can also upload a single file by setting `local_path` to point to a file on your machine. If that's the case, `path_in_repo` is optional and will default to the name of your local file:
+
+```bash
+>>> huggingface-cli upload Wauplin/my-cool-dataset ./files/train.csv --repo-type dataset
+https://huggingface.co/datasetsWauplin/my-cool-dataset/blob/main/train.csv
+```
+
+If you want to upload a single file to a specific directory, set `path_in_repo` accordingly:
+
+```bash
+>>> huggingface-cli upload Wauplin/my-cool-dataset ./files/train.csv /data/train.csv --repo-type dataset
+https://huggingface.co/datasetsWauplin/my-cool-dataset/blob/main/data/train.csv
+```
+
+### Upload multiple files
+
+To upload multiple files from a folder at once without uploading the entire folder, use the `--include` and `--exclude` patterns. It can also be combined with the `--delete` option to delete files on the repo while uploading new ones. In the example below, we sync the local Space by deleting remote files and uploading all CSV files:
+
+```bash
+# Sync local Space with Hub (upload new CSV files, delete removed files)
+>>> huggingface-cli upload Wauplin/my-cool-dataset --repo-type dataset --include="/data/*.csv" --delete="*" --commit-message="Sync local dataset with Hub"
+...
+```
+
+### Upload to an organization
+
+To upload content to a repo owned by an organization instead of a personal repo, you must explicitly specify it in the `repo_id`:
+
+```bash
+>>> huggingface-cli upload MyCoolOrganization/my-cool-dataset . . --repo-type dataset
+https://huggingface.co/datasetsMyCoolOrganization/my-cool-dataset/tree/main/
+```
+
+### Upload to a specific revision
+
+By default, files are uploaded to the `main` branch. If you want to upload files to another branch or reference, use the `--revision` option:
+
+```bash
+# Upload files to a PR
+huggingface-cli upload bigcode/the-stack . . --repo-type dataset --revision refs/pr/104
+...
+```
+
+**Note:** if `revision` does not exist and `--create-pr` is not set, a branch will be created automatically from the `main` branch.
+
+### Upload and create a PR
+
+If you don't have the permission to push to a repo, you must open a PR and let the authors know about the changes you want to make. This can be done by setting the `--create-pr` option:
+
+```bash
+# Create a PR and upload the files to it
+>>> huggingface-cli upload bigcode/the-stack --repo-type dataset --revision refs/pr/104 --create-pr . .
+https://huggingface.co/datasets/bigcode/the-stack/blob/refs%2Fpr%2F104/
+```
+
+### Upload at regular intervals
+
+In some cases, you might want to push regular updates to a repo. For example, this is useful if your dataset is growing over time and you want to upload the data folder every 10 minutes. You can do this using the `--every` option:
+
+```bash
+# Upload new logs every 10 minutes
+huggingface-cli upload my-cool-dynamic-dataset data/ --every=10
+```
+
+### Specify a commit message
+
+Use the `--commit-message` and `--commit-description` to set a custom message and description for your commit instead of the default one
+
+```bash
+>>> huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type dataset --commit-message="Version 2" --commit-description="Train size: 4321. Check Dataset Viewer for more details."
+...
+https://huggingface.co/datasetsWauplin/my-cool-dataset/tree/main
+```
+
+### Specify a token
+
+To upload files, you must use a token. By default, the token saved locally (using `huggingface-cli login`) will be used. If you want to authenticate explicitly, use the `--token` option:
+
+```bash
+>>> huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type dataset --token=hf_****
+...
+https://huggingface.co/datasetsWauplin/my-cool-data/tree/main
+```
+
+### Quiet mode
+
+By default, the `huggingface-cli upload` command will be verbose. It will print details such as warning messages, information about the uploaded files, and progress bars. If you want to silence all of this, use the `--quiet` option. Only the last line (i.e. the URL to the uploaded files) is printed. This can prove useful if you want to pass the output to another command in a script.
+
+```bash
+>>> huggingface-cli upload Wauplin/my-cool-dataset ./data . --repo-type dataset --quiet
+https://huggingface.co/datasets/Wauplin/my-cool-dataset/tree/main
+```
+
+## Enjoy !
+
+Congratulations, your dataset has now been uploaded to the Hugging Face Hub where anyone can load it in a single line of code! 🥳
+
+```
+dataset = load_dataset("Wauplin/my-cool-dataset")
+```
+
+If your dataset is supported, it should also have a [Dataset Viewer](https://huggingface.co/docs/hub/datasets-viewer) for everyone to explore the dataset content.
+
+Finally, don't forget to enrich the dataset card to document your dataset and make it discoverable! Check out the [Create a dataset card](dataset_card) guide to learn more.

docs/source/stream.mdx

@@ -0,0 +1,628 @@
+# Stream
+
+Dataset streaming lets you work with a dataset without downloading it.
+The data is streamed as you iterate over the dataset.
+This is especially helpful when:
+
+- You don't want to wait for an extremely large dataset to download.
+- The dataset size exceeds the amount of available disk space on your computer.
+- You want to quickly explore just a few samples of a dataset.
+
+<div class="flex justify-center">
+    <img class="block dark:hidden" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/streaming.gif"/>
+    <img class="hidden dark:block" src="https://huggingface.co/datasets/huggingface/documentation-images/resolve/main/datasets/streaming-dark.gif"/>
+</div>
+
+For example, the English split of the [HuggingFaceFW/fineweb](https://huggingface.co/datasets/HuggingFaceFW/fineweb) dataset is 45 terabytes, but you can use it instantly with streaming. Stream a dataset by setting `streaming=True` in [`load_dataset`] as shown below:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('HuggingFaceFW/fineweb', split='train', streaming=True)
+>>> print(next(iter(dataset)))
+{'text': 'How AP reported in all formats from tornado-stricken regionsMarch 8, 2012\nWhen the first serious bout of tornadoes of 2012 blew through middle America in the middle of the night, they touched down in places hours from any AP bureau...', ...,
+ 'language_score': 0.9721424579620361, 'token_count': 717}
+```
+
+Dataset streaming also lets you work with a dataset made of local files without doing any conversion.
+In this case, the data is streamed from the local files as you iterate over the dataset.
+This is especially helpful when:
+
+- You don't want to wait for an extremely large local dataset to be converted to Arrow.
+- The converted files size would exceed the amount of available disk space on your computer.
+- You want to quickly explore just a few samples of a dataset.
+- You want to load only certain columns or efficiently filter a Parquet dataset.
+
+For example, you can stream a local dataset of hundreds of compressed JSONL files like [oscar-corpus/OSCAR-2201](https://huggingface.co/datasets/oscar-corpus/OSCAR-2201) to use it instantly:
+
+```py
+>>> from datasets import load_dataset
+>>> data_files = {'train': 'path/to/OSCAR-2201/compressed/en_meta/*.jsonl.gz'}
+>>> dataset = load_dataset('json', data_files=data_files, split='train', streaming=True)
+>>> print(next(iter(dataset)))
+{'id': 0, 'text': 'Founded in 2015, Golden Bees is a leading programmatic recruitment platform dedicated to employers, HR agencies and job boards. The company has developed unique HR-custom technologies and predictive algorithms to identify and attract the best candidates for a job opportunity.', ...
+```
+
+Parquet is a columnar format that allows you to stream and load only a subset of columns and ignore unwanted columns. Parquet also stores metadata such as column statistics (at the file and row group level), enabling efficient filtering. Use the `columns` and `filters` arguments of [`datasets.packaged_modules.parquet.ParquetConfig`] to stream Parquet datasets, select columns, and apply filters:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('HuggingFaceFW/fineweb', split='train', streaming=True, columns=["url", "date"])
+>>> print(next(iter(dataset)))
+{'url': 'http://%20jwashington@ap.org/Content/Press-Release/2012/How-AP-reported-in-all-formats-from-tornado-stricken-regions', 'date': '2013-05-18T05:48:54Z'}
+>>> dataset = load_dataset('HuggingFaceFW/fineweb', split='train', streaming=True, filters=[("language_score", ">=", 0.99)])
+>>> print(next(iter(dataset)))
+{'text': 'Everyone wishes for something. And lots of people believe they know how to make their wishes come true with magical thinking.\nWhat is it? "Magical thinking is a belief in forms of causation, with no known physical basis," said Professor Emily Pronin of Princeton...', ...,
+ 'language_score': 0.9900368452072144, 'token_count': 716}
+```
+
+Loading a dataset in streaming mode creates a new dataset type instance (instead of the classic [`Dataset`] object), known as an [`IterableDataset`].
+This special type of dataset has its own set of processing methods shown below.
+
+> [!TIP]
+> An [`IterableDataset`] is useful for iterative jobs like training a model.
+> You shouldn't use a [`IterableDataset`] for jobs that require random access to examples because you have to iterate all over it using a for loop. Getting the last example in an iterable dataset would require you to iterate over all the previous examples.
+> You can find more details in the [Dataset vs. IterableDataset guide](./about_mapstyle_vs_iterable).
+
+
+## Column indexing
+
+Sometimes it is convenient to iterate over values of a specific column. Fortunately, an [`IterableDataset`] supports column indexing:
+```python
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("allenai/c4", "en", streaming=True, split="train")
+>>> print(next(iter(dataset["text"])))
+Beginners BBQ Class Taking Place in Missoula!...
+```
+
+## Convert from a Dataset
+
+If you have an existing [`Dataset`] object, you can convert it to an [`IterableDataset`] with the [`~Dataset.to_iterable_dataset`] function. This is actually faster than setting the `streaming=True` argument in [`load_dataset`] because the data is streamed from local files.
+
+```py
+>>> from datasets import load_dataset
+
+# faster 🐇
+>>> dataset = load_dataset("ethz/food101")
+>>> iterable_dataset = dataset.to_iterable_dataset()
+
+# slower 🐢
+>>> iterable_dataset = load_dataset("ethz/food101", streaming=True)
+```
+
+The [`~Dataset.to_iterable_dataset`] function supports sharding when the [`IterableDataset`] is instantiated. This is useful when working with big datasets, and you'd like to shuffle the dataset or to enable fast parallel loading with a PyTorch DataLoader.
+
+```py
+>>> import torch
+>>> from datasets import load_dataset
+
+>>> dataset = load_dataset("ethz/food101")
+>>> iterable_dataset = dataset.to_iterable_dataset(num_shards=64) # shard the dataset
+>>> iterable_dataset = iterable_dataset.shuffle(buffer_size=10_000)  # shuffles the shards order and use a shuffle buffer when you start iterating
+dataloader = torch.utils.data.DataLoader(iterable_dataset, num_workers=4)  # assigns 64 / 4 = 16 shards from the shuffled list of shards to each worker when you start iterating
+```
+
+## Shuffle
+
+Like a regular [`Dataset`] object, you can also shuffle a [`IterableDataset`] with [`IterableDataset.shuffle`].
+
+The `buffer_size` argument controls the size of the buffer to randomly sample examples from. Let's say your dataset has one million examples, and you set the `buffer_size` to ten thousand. [`IterableDataset.shuffle`] will randomly select examples from the first ten thousand examples in the buffer. Selected examples in the buffer are replaced with new examples. By default, the buffer size is 1,000.
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('HuggingFaceFW/fineweb', split='train', streaming=True)
+>>> shuffled_dataset = dataset.shuffle(seed=42, buffer_size=10_000)
+```
+
+> [!TIP]
+> 
+> [`IterableDataset.shuffle`] fills the buffer using up to `max_buffer_input_shards` shards at a time (default is 10), which greatly improves the quality of the shuffling. It also shuffles the order of the shards if the dataset is sharded into multiple files.
+
+> [!TIP]
+>
+> If you're streaming a Parquet dataset with only one or few files, use [`~IterableDataset.reshard`] before shuffling. This is useful to shard the dataset by row group instead of by file, improving shuffling quality.
+
+## Reshuffle
+
+Sometimes you may want to reshuffle the dataset after each epoch. This will require you to set a different seed for each epoch. Use [`IterableDataset.set_epoch`] in between epochs to tell the dataset what epoch you're on.
+
+Your seed effectively becomes: `initial seed + current epoch`.
+
+```py
+>>> for epoch in range(epochs):
+...     shuffled_dataset.set_epoch(epoch)
+...     for example in shuffled_dataset:
+...         ...
+```
+
+## Split dataset
+
+You can split your dataset one of two ways:
+
+- [`IterableDataset.take`] returns the first `n` examples in a dataset:
+
+```py
+>>> dataset = load_dataset('HuggingFaceFW/fineweb', split='train', streaming=True)
+>>> dataset_head = dataset.take(2)
+>>> list(dataset_head)
+[{'text': "How AP reported in all formats from tor...},
+ {'text': 'Did you know you have two little yellow...}]
+```
+
+- [`IterableDataset.skip`] omits the first `n` examples in a dataset and returns the remaining examples:
+
+```py
+>>> train_dataset = shuffled_dataset.skip(1000)
+```
+
+> [!WARNING]
+> `take` and `skip` prevent future calls to `shuffle` because they lock in the order of the shards. You should `shuffle` your dataset before splitting it.
+
+<a id='interleave_datasets'></a>
+
+
+### Shard
+
+🤗 Datasets supports sharding to divide a very large dataset into a predefined number of chunks. Specify the `num_shards` parameter in [`~IterableDataset.shard`] to determine the number of shards to split the dataset into. You'll also need to provide the shard you want to return with the `index` parameter.
+
+For example, the [amazon_polarity](https://huggingface.co/datasets/fancyzhx/amazon_polarity) dataset has 4 shards (in this case they are 4 Parquet files):
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("fancyzhx/amazon_polarity", split="train", streaming=True)
+>>> print(dataset)
+IterableDataset({
+    features: ['label', 'title', 'content'],
+    num_shards: 4
+})
+```
+
+After sharding the dataset into two chunks, the first one will only have 2 shards:
+
+```py
+>>> dataset.shard(num_shards=2, index=0)
+IterableDataset({
+    features: ['label', 'title', 'content'],
+    num_shards: 2
+})
+```
+
+To increase the number of shards of a dataset, you can use [`IterableDataset.reshard`]:
+
+```py
+>>> dataset.reshard()
+IterableDataset({
+    features: ['label', 'title', 'content'],
+    num_shards: 3600
+})
+```
+
+The resharding mechanism depends on the dataset file format.
+For example for Parquet, it reshards using row groups instead of having one file per shard.
+See how it works for every format in [`IterableDataset.reshard`]'s documentation.
+
+If your dataset has `dataset.num_shards==1` even after resharding, you should chunk it using [`IterableDataset.skip`] and [`IterableDataset.take`] instead.
+
+## Concatenate
+
+Separate datasets can be concatenated if they share the same column types. Concatenate datasets with [`concatenate_datasets`]:
+
+```py
+>>> from datasets import concatenate_datasets, load_dataset
+
+>>> stories = load_dataset("ajibawa-2023/General-Stories-Collection", split="train", streaming=True)
+>>> stories = stories.select_columns(["text"])  # only keep the 'text' column
+IterableDataset({
+    features: Unknown,
+    num_shards: 10
+})
+
+>>> wiki = load_dataset("wikimedia/wikipedia", "20231101.en", split="train", streaming=True)
+>>> wiki = wiki.select_columns(["text"])  # only keep the 'text' column
+IterableDataset({
+    features: ['text'],
+    num_shards: 41
+})
+
+>>> bert_dataset = concatenate_datasets([stories, wiki])
+IterableDataset({
+    features: ['text'],
+    num_shards: 51
+})
+```
+
+The shards of the concatenated dataset are the shards of the input datasets.
+
+You can also concatenate two datasets horizontally by setting `axis=1` as long as the datasets have the same number of rows:
+
+```py
+>>> from datasets import IterableDataset
+>>> stories_ids = IterableDataset.from_dict({"ids": list(range(num_stories))})
+>>> stories_with_ids = concatenate_datasets([stories, stories_ids], axis=1)
+```
+
+In this case, the concatenated dataset only has 1 shard, to avoid ending with unaligned shards from the input datasets.
+
+## Interleave
+
+[`interleave_datasets`] can combine an [`IterableDataset`] with other datasets if they have the same column types. The combined dataset returns alternating examples from each of the original datasets.
+
+```py
+>>> from datasets import interleave_datasets
+>>> es_dataset = load_dataset('allenai/c4', 'es', split='train', streaming=True)
+IterableDataset({
+    features: Unknown,
+    num_shards: 2048
+})
+>>> fr_dataset = load_dataset('allenai/c4', 'fr', split='train', streaming=True)
+IterableDataset({
+    features: Unknown,
+    num_shards: 2048
+})
+
+>>> multilingual_dataset = interleave_datasets([es_dataset, fr_dataset])
+IterableDataset({
+    features: ['text', 'timestamp', 'url'],
+    num_shards: 2048
+})
+>>> list(multilingual_dataset.take(2))
+[{'text': 'Comprar Zapatillas para niña en chancla con goma por...'},
+ {'text': 'Le sacre de philippe ier, 23 mai 1059 - Compte Rendu...'}]
+```
+
+Define sampling probabilities from each of the original datasets for more control over how each of them are sampled and combined. Set the `probabilities` argument with your desired sampling probabilities:
+
+```py
+>>> multilingual_dataset_with_oversampling = interleave_datasets([es_dataset, fr_dataset], probabilities=[0.8, 0.2], seed=42)
+>>> list(multilingual_dataset_with_oversampling.take(2))
+[{'text': 'Comprar Zapatillas para niña en chancla con goma por...'},
+ {'text': 'Chevrolet Cavalier Usados en Bogota - Carros en Vent...'}]
+```
+
+Around 80% of the final dataset is made of the `es_dataset`, and 20% of the `fr_dataset`.
+
+You can also specify the `stopping_strategy`. The default strategy, `first_exhausted`, is a subsampling strategy, i.e the dataset construction is stopped as soon one of the dataset runs out of samples.
+You can specify `stopping_strategy=all_exhausted` to execute an oversampling strategy. In this case, the dataset construction is stopped as soon as every samples in every dataset has been added at least once. In practice, it means that if a dataset is exhausted, it will return to the beginning of this dataset until the stop criterion has been reached.
+Note that if no sampling probabilities are specified, the new dataset will have `max_length_datasets*nb_dataset samples`.
+There is also `stopping_strategy=all_exhausted_without_replacement` to ensure that every sample is seen exactly once.
+
+To ensure proper parallelism using sharding, the shards of the interleaved dataset contain at least 1 shard of every input dataset. Therefore the sharding level of the interleaved dataset is the minimum sharding level of the input datasets.
+
+E.g. if the input datasets have respectively 32, 48 and 128 shards, then the interleaved dataset has 32 = min(32, 48, 128) shards, and each new shard has 1 shard from the first dataset, 1-2 shards from the second dataset and 4 shards from the third dataset.
+
+## Rename, remove, and cast
+
+The following methods allow you to modify the columns of a dataset. These methods are useful for renaming or removing columns and changing columns to a new set of features.
+
+### Rename
+
+Use [`IterableDataset.rename_column`] when you need to rename a column in your dataset. Features associated with the original column are actually moved under the new column name, instead of just replacing the original column in-place.
+
+Provide [`IterableDataset.rename_column`] with the name of the original column, and the new column name:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('allenai/c4', 'en', streaming=True, split='train')
+>>> dataset = dataset.rename_column("text", "content")
+```
+
+### Remove
+
+When you need to remove one or more columns, give [`IterableDataset.remove_columns`] the name of the column to remove. Remove more than one column by providing a list of column names:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('allenai/c4', 'en', streaming=True, split='train')
+>>> dataset = dataset.remove_columns('timestamp')
+```
+
+### Cast
+
+[`IterableDataset.cast`] changes the feature type of one or more columns. This method takes your new `Features` as its argument. The following sample code shows how to change the feature types of `ClassLabel` and `Value`:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('nyu-mll/glue', 'mrpc', split='train', streaming=True)
+>>> dataset.features
+{'sentence1': Value('string'),
+'sentence2': Value('string'),
+'label': ClassLabel(names=['not_equivalent', 'equivalent']),
+'idx': Value('int32')}
+
+>>> from datasets import ClassLabel, Value
+>>> new_features = dataset.features.copy()
+>>> new_features["label"] = ClassLabel(names=['negative', 'positive'])
+>>> new_features["idx"] = Value('int64')
+>>> dataset = dataset.cast(new_features)
+>>> dataset.features
+{'sentence1': Value('string'),
+'sentence2': Value('string'),
+'label': ClassLabel(names=['negative', 'positive']),
+'idx': Value('int64')}
+```
+
+> [!TIP]
+> Casting only works if the original feature type and new feature type are compatible. For example, you can cast a column with the feature type `Value('int32')` to `Value('bool')` if the original column only contains ones and zeros.
+
+Use [`IterableDataset.cast_column`] to change the feature type of just one column. Pass the column name and its new feature type as arguments:
+
+```py
+>>> dataset.features
+{'audio': Audio(sampling_rate=44100, mono=True)}
+
+>>> dataset = dataset.cast_column("audio", Audio(sampling_rate=16000))
+>>> dataset.features
+{'audio': Audio(sampling_rate=16000, mono=True)}
+```
+
+## Map
+
+Similar to the [`Dataset.map`] function for a regular [`Dataset`], 🤗  Datasets features [`IterableDataset.map`] for processing an [`IterableDataset`].
+[`IterableDataset.map`] applies processing on-the-fly when examples are streamed.
+
+It allows you to apply a processing function to each example in a dataset, independently or in batches. This function can even create new rows and columns.
+
+The following example demonstrates how to tokenize a [`IterableDataset`]. The function needs to accept and output a `dict`:
+
+```py
+>>> def add_prefix(example):
+...     example['text'] = 'My text: ' + example['text']
+...     return example
+```
+
+Next, apply this function to the dataset with [`IterableDataset.map`]:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('allenai/c4', 'en', streaming=True, split='train')
+>>> updated_dataset = dataset.map(add_prefix)
+>>> list(updated_dataset.take(3))
+[{'text': 'My text: Beginners BBQ Class Taking Place in Missoula!\nDo you want to get better at making...',
+  'timestamp': '2019-04-25 12:57:54',
+  'url': 'https://klyq.com/beginners-bbq-class-taking-place-in-missoula/'},
+ {'text': 'My text: Discussion in \'Mac OS X Lion (10.7)\' started by axboi87, Jan 20, 2012.\nI\'ve go...',
+  'timestamp': '2019-04-21 10:07:13',
+  'url': 'https://forums.macrumors.com/threads/restore-from-larger-disk-to-smaller-disk.1311329/'},
+ {'text': 'My text: Foil plaid lycra and spandex shortall with metallic slinky insets. Attached metall...',
+  'timestamp': '2019-04-25 10:40:23',
+  'url': 'https://awishcometrue.com/Catalogs/Clearance/Tweens/V1960-Find-A-Way'}]
+```
+
+Let's take a look at another example, except this time, you will remove columns with [`IterableDataset.map`]. When you remove a column, it is only removed after the example has been provided to the mapped function. This allows the mapped function to use the content of the columns before they are removed.
+
+Specify the column to remove with the `remove_columns` argument in [`IterableDataset.map`]:
+
+```py
+>>> updated_dataset = dataset.map(add_prefix, remove_columns=["timestamp", "url"])
+>>> list(updated_dataset.take(3))
+[{'text': 'My text: Beginners BBQ Class Taking Place in Missoula!\nDo you want to get better at making...'},
+ {'text': 'My text: Discussion in \'Mac OS X Lion (10.7)\' started by axboi87, Jan 20, 2012.\nI\'ve go...'},
+ {'text': 'My text: Foil plaid lycra and spandex shortall with metallic slinky insets. Attached metall...'}]
+```
+
+### Batch processing
+
+[`IterableDataset.map`] also supports working with batches of examples. Operate on batches by setting `batched=True`. The default batch size is 1000, but you can adjust it with the `batch_size` argument. This opens the door to many interesting applications such as tokenization, splitting long sentences into shorter chunks, and data augmentation.
+
+#### Tokenization
+
+```py
+>>> from datasets import load_dataset
+>>> from transformers import AutoTokenizer
+>>> dataset = load_dataset("allenai/c4", "en", streaming=True, split="train")
+>>> tokenizer = AutoTokenizer.from_pretrained('distilbert-base-uncased')
+>>> def encode(examples):
+...     return tokenizer(examples['text'], truncation=True, padding='max_length')
+>>> dataset = dataset.map(encode, batched=True, remove_columns=["text", "timestamp", "url"])
+>>> next(iter(dataset))
+{'input_ids': [101, 4088, 16912, 22861, 4160, 2465, 2635, 2173, 1999, 3335, ..., 0, 0, 0],
+'attention_mask': [1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, 1, ..., 0, 0]}
+```
+
+> [!TIP]
+> See other examples of batch processing in the [batched map processing](./process#batch-processing) documentation. They work the same for iterable datasets.
+
+### Filter
+
+You can filter rows in the dataset based on a predicate function using [`Dataset.filter`]. It returns rows that match a specified condition:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset('HuggingFaceFW/fineweb', streaming=True, split='train')
+>>> start_with_ar = dataset.filter(lambda example: example['text'].startswith('San Francisco'))
+>>> next(iter(start_with_ar))
+{'text': 'San Francisco 49ers cornerback Shawntae Spencer will miss the rest of the sea...}
+```
+
+[`Dataset.filter`] can also filter by indices if you set `with_indices=True`:
+
+```py
+>>> even_dataset = dataset.filter(lambda example, idx: idx % 2 == 0, with_indices=True)
+>>> list(even_dataset.take(3))
+[{'text': 'How AP reported in all formats from tornado-stricken regionsMarch 8, 2012 Whe...},
+ {'text': 'Car Wash For Clara! Now is your chance to help! 2 year old Clara Woodward has...},
+ {'text': 'Log In Please enter your ECode to log in. Forgotten your eCode? If you create...}]
+```
+
+## Batch
+
+The `batch` method transforms your `IterableDataset` into an iterable of batches. This is particularly useful when you want to work with batches in your training loop or when using frameworks that expect batched inputs.
+
+> [!TIP]
+> There is also a "Batch Processing" option when using the `map` function to apply a function to batches of data, which is discussed in the [Map section](#map) above. The `batch` method described here is different and provides a more direct way to create batches from your dataset.
+
+You can use the `batch` method like this:
+
+```python
+from datasets import load_dataset
+
+# Load a dataset in streaming mode
+dataset = load_dataset("some_dataset", split="train", streaming=True)
+
+# Create batches of 32 samples
+batched_dataset = dataset.batch(batch_size=32)
+
+# Iterate over the batched dataset
+for batch in batched_dataset:
+    print(batch)
+    break
+```
+
+In this example, batched_dataset is still an IterableDataset, but each item yielded is now a batch of 32 samples instead of a single sample.
+This batching is done on-the-fly as you iterate over the dataset, preserving the memory-efficient nature of IterableDataset.
+
+The batch method also provides a drop_last_batch parameter. 
+When set to True, it will discard the last batch if it's smaller than the specified batch_size. 
+This can be useful in scenarios where your downstream processing requires all batches to be of the same size:
+
+```python
+batched_dataset = dataset.batch(batch_size=32, drop_last_batch=True)
+```
+
+## Stream in a training loop
+
+[`IterableDataset`] can be integrated into a training loop. First, shuffle the dataset:
+
+<frameworkcontent>
+<pt>
+```py
+>>> seed, buffer_size = 42, 10_000
+>>> dataset = dataset.shuffle(seed, buffer_size=buffer_size)
+```
+
+Lastly, create a simple training loop and start training:
+
+```py
+>>> import torch
+>>> from torch.utils.data import DataLoader
+>>> from transformers import AutoModelForMaskedLM, DataCollatorForLanguageModeling
+>>> from tqdm import tqdm
+>>> dataset = dataset.with_format("torch")
+>>> dataloader = DataLoader(dataset, collate_fn=DataCollatorForLanguageModeling(tokenizer))
+>>> device = 'cuda' if torch.cuda.is_available() else 'cpu' 
+>>> model = AutoModelForMaskedLM.from_pretrained("distilbert-base-uncased")
+>>> model.train().to(device)
+>>> optimizer = torch.optim.AdamW(params=model.parameters(), lr=1e-5)
+>>> for epoch in range(3):
+...     dataset.set_epoch(epoch)
+...     for i, batch in enumerate(tqdm(dataloader, total=5)):
+...         if i == 5:
+...             break
+...         batch = {k: v.to(device) for k, v in batch.items()}
+...         outputs = model(**batch)
+...         loss = outputs[0]
+...         loss.backward()
+...         optimizer.step()
+...         optimizer.zero_grad()
+...         if i % 10 == 0:
+...             print(f"loss: {loss}")
+```
+</pt>
+</frameworkcontent>
+
+<!-- TODO: Write the TF content! -->
+
+### Save a dataset checkpoint and resume iteration
+
+If your training loop stops, you may want to restart the training from where it was. To do so you can save a checkpoint of your model and optimizers, as well as your data loader.
+
+Iterable datasets don't provide random access to a specific example index to resume from, but you can use [`IterableDataset.state_dict`] and [`IterableDataset.load_state_dict`] to resume from a checkpoint instead, similarly to what you can do for models and optimizers:
+
+```python
+>>> iterable_dataset = Dataset.from_dict({"a": range(6)}).to_iterable_dataset(num_shards=3)
+>>> for idx, example in enumerate(iterable_dataset):
+...     print(example)
+...     if idx == 2:
+...         state_dict = iterable_dataset.state_dict()
+...         print("checkpoint")
+...         break
+>>> iterable_dataset.load_state_dict(state_dict)
+>>> print(f"restart from checkpoint")
+>>> for example in iterable_dataset:
+...     print(example)
+```
+
+Returns:
+
+```
+{'a': 0}
+{'a': 1}
+{'a': 2}
+checkpoint
+restart from checkpoint
+{'a': 3}
+{'a': 4}
+{'a': 5}
+```
+
+Under the hood, the iterable dataset keeps track of the current shard being read and the example index in the current shard and it stores this info in the `state_dict`.
+
+To resume from a checkpoint, the dataset skips all the shards that were previously read to restart from the current shard. 
+Then it reads the shard and skips examples until it reaches the exact example from the checkpoint.
+
+Therefore restarting a dataset is quite fast, since it will not re-read the shards that have already been iterated on. Still, resuming a dataset is generally not instantaneous since it has to restart reading from the beginning of the current shard and skip examples until it reaches the checkpoint location.
+
+This can be used with the `StatefulDataLoader` from `torchdata`:
+
+```python
+>>> from torchdata.stateful_dataloader import StatefulDataLoader
+>>> iterable_dataset = load_dataset("deepmind/code_contests", streaming=True, split="train")
+>>> dataloader = StatefulDataLoader(iterable_dataset, batch_size=32, num_workers=4)
+>>> # checkpoint
+>>> state_dict = dataloader.state_dict()  # uses iterable_dataset.state_dict() under the hood
+>>> # resume from checkpoint
+>>> dataloader.load_state_dict(state_dict)  # uses iterable_dataset.load_state_dict() under the hood
+```
+
+> [!TIP]
+> Resuming returns exactly where the checkpoint was saved except if `.shuffle()` is used: examples from shuffle buffers are lost when resuming and the buffers are refilled with new data.
+
+
+## Save
+
+Once your iterable dataset is ready, you can save it as a Hugging Face Dataset in Parquet format and reuse it later with [`load_dataset`].
+
+Save your dataset by providing the name of the dataset repository on Hugging Face you wish to save it to as an argument to [`~Dataset.push_to_hub`]. This iterates over the dataset and progressively uploads the data to Hugging Face:
+
+```python
+dataset.push_to_hub("username/my_dataset")
+```
+
+If the dataset consists of multiple shards (`dataset.num_shards > 1`), you can use multiple processes to upload it in parallel. This is especially useful if you applied `map()` or `filter()` steps since they will run faster in parallel:
+
+```python
+dataset.push_to_hub("username/my_dataset", num_proc=8)
+```
+
+Use the [`load_dataset`] function to reload the dataset:
+
+```python
+from datasets import load_dataset
+reloaded_dataset = load_dataset("username/my_dataset")
+```
+
+## Export
+
+🤗 Datasets supports exporting as well so you can work with your dataset in other applications. The following table shows currently supported file formats you can export to:
+
+| File type               | Export method                                                  |
+|-------------------------|----------------------------------------------------------------|
+| CSV                     | [`IterableDataset.to_csv`]                                    |
+| JSON                    | [`IterableDataset.to_json`]                                   |
+| Parquet                 | [`IterableDataset.to_parquet`]                                |
+| SQL                     | [`IterableDataset.to_sql`]                                    |
+| In-memory Python object | [`IterableDataset.to_pandas`], [`IterableDataset.to_polars`] or [`IterableDataset.to_dict`] |
+
+For example, export your dataset to a CSV file like this:
+
+```py
+>>> dataset.to_csv("path/of/my/dataset.csv")
+```
+
+If you have a large dataset, you can save one file per shard, e.g.
+
+```py
+>>> num_shards = dataset.num_shards
+>>> for index in range(num_shards):
+...     shard = dataset.shard(index, num_shards)
+...     shard.to_parquet(f"path/of/my/dataset/data-{index:05d}.parquet")
+```

docs/source/tabular_load.mdx

@@ -0,0 +1,151 @@
+# Load tabular data
+
+A tabular dataset is a generic dataset used to describe any data stored in rows and columns, where the rows represent an example and the columns represent a feature (can be continuous or categorical). These datasets are commonly stored in CSV files, Pandas DataFrames, and in database tables. This guide will show you how to load and create a tabular dataset from:
+
+- CSV files
+- Pandas DataFrames
+- HDF5 files
+- Databases
+
+## CSV files
+
+🤗 Datasets can read CSV files by specifying the generic `csv` dataset builder name in the [`~datasets.load_dataset`] method. To load more than one CSV file, pass them as a list to the `data_files` parameter:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("csv", data_files="my_file.csv")
+
+# load multiple CSV files
+>>> dataset = load_dataset("csv", data_files=["my_file_1.csv", "my_file_2.csv", "my_file_3.csv"])
+```
+
+You can also map specific CSV files to the train and test splits:
+
+```py
+>>> dataset = load_dataset("csv", data_files={"train": ["my_train_file_1.csv", "my_train_file_2.csv"], "test": "my_test_file.csv"})
+```
+
+To load remote CSV files, pass the URLs instead:
+
+```py
+>>> base_url = "https://huggingface.co/datasets/lhoestq/demo1/resolve/main/data/"
+>>> dataset = load_dataset('csv', data_files={"train": base_url + "train.csv", "test": base_url + "test.csv"})
+```
+
+To load zipped CSV files:
+
+```py
+>>> url = "https://domain.org/train_data.zip"
+>>> data_files = {"train": url}
+>>> dataset = load_dataset("csv", data_files=data_files)
+```
+
+## Pandas DataFrames
+
+🤗 Datasets also supports loading datasets from [Pandas DataFrames](https://pandas.pydata.org/docs/reference/api/pandas.DataFrame.html) with the [`~datasets.Dataset.from_pandas`] method:
+
+```py
+>>> from datasets import Dataset
+>>> import pandas as pd
+
+# create a Pandas DataFrame
+>>> df = pd.read_csv("https://huggingface.co/datasets/imodels/credit-card/raw/main/train.csv")
+>>> df = pd.DataFrame(df)
+# load Dataset from Pandas DataFrame
+>>> dataset = Dataset.from_pandas(df)
+```
+
+Use the `splits` parameter to specify the name of the dataset split:
+
+```py
+>>> train_ds = Dataset.from_pandas(train_df, split="train")
+>>> test_ds = Dataset.from_pandas(test_df, split="test")
+```
+
+If the dataset doesn't look as expected, you should explicitly [specify your dataset features](loading#specify-features). A [pandas.Series](https://pandas.pydata.org/docs/reference/api/pandas.Series.html) may not always carry enough information for Arrow to automatically infer a data type. For example, if a DataFrame is of length `0` or if the Series only contains `None/NaN` objects, the type is set to `null`.
+
+## HDF5 files
+
+[HDF5](https://www.hdfgroup.org/solutions/hdf5/) files are commonly used for storing large amounts of numerical data in scientific computing and machine learning. Loading HDF5 files with 🤗 Datasets is similar to loading CSV files:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("hdf5", data_files="data.h5")
+```
+
+Note that the HDF5 loader assumes that the file has "tabular" structure, i.e. that all datasets in the file have (the same number of) rows on their first dimension.
+
+## Databases
+
+Datasets stored in databases are typically accessed with SQL queries. With 🤗 Datasets, you can connect to a database, query for the data you need, and create a dataset out of it. Then you can use all the processing features of 🤗 Datasets to prepare your dataset for training.
+
+### SQLite
+
+SQLite is a small, lightweight database that is fast and easy to set up. You can use an existing database if you'd like, or follow along and start from scratch.
+
+Start by creating a quick SQLite database with this [Covid-19 data](https://github.com/nytimes/covid-19-data/blob/master/us-states.csv) from the New York Times:
+
+```py
+>>> import sqlite3
+>>> import pandas as pd
+
+>>> conn = sqlite3.connect("us_covid_data.db")
+>>> df = pd.read_csv("https://raw.githubusercontent.com/nytimes/covid-19-data/master/us-states.csv")
+>>> df.to_sql("states", conn, if_exists="replace")
+```
+
+This creates a `states` table in the `us_covid_data.db` database which you can now load into a dataset.
+
+To connect to the database, you'll need the [URI string](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) that identifies your database. Connecting to a database with a URI caches the returned dataset. The URI string differs for each database dialect, so be sure to check the [Database URLs](https://docs.sqlalchemy.org/en/13/core/engines.html#database-urls) for whichever database you're using.
+
+For SQLite, it is:
+
+```py
+>>> uri = "sqlite:///us_covid_data.db"
+```
+
+Load the table by passing the table name and URI to [`~datasets.Dataset.from_sql`]:
+
+```py
+>>> from datasets import Dataset
+
+>>> ds = Dataset.from_sql("states", uri)
+>>> ds
+Dataset({
+    features: ['index', 'date', 'state', 'fips', 'cases', 'deaths'],
+    num_rows: 54382
+})
+```
+
+Then you can use all of 🤗 Datasets process features like [`~datasets.Dataset.filter`] for example:
+
+```py
+>>> ds.filter(lambda x: x["state"] == "California")
+```
+
+You can also load a dataset from a SQL query instead of an entire table, which is useful for querying and joining multiple tables. 
+
+Load the dataset by passing your query and URI to [`~datasets.Dataset.from_sql`]:
+
+```py
+>>> from datasets import Dataset
+
+>>> ds = Dataset.from_sql('SELECT * FROM states WHERE state="California";', uri)
+>>> ds
+Dataset({
+    features: ['index', 'date', 'state', 'fips', 'cases', 'deaths'],
+    num_rows: 1019
+})
+```
+
+Then you can use all of 🤗 Datasets process features like [`~datasets.Dataset.filter`] for example:
+
+```py
+>>> ds.filter(lambda x: x["cases"] > 10000)
+```
+
+### PostgreSQL
+
+You can also connect and load a dataset from a PostgreSQL database, however we won't directly demonstrate how in the documentation because the example is only meant to be run in a notebook. Instead, take a look at how to install and setup a PostgreSQL server in this [notebook](https://colab.research.google.com/github/nateraw/huggingface-hub-examples/blob/main/sql_with_huggingface_datasets.ipynb#scrollTo=d83yGQMPHGFi)!
+
+After you've setup your PostgreSQL database, you can use the [`~datasets.Dataset.from_sql`] method to load a dataset from a table or query.

docs/source/troubleshoot.mdx

@@ -0,0 +1,118 @@
+# Troubleshooting
+
+This guide aims to provide you the tools and knowledge required to navigate some common issues. If the suggestions listed 
+in this guide do not cover your such situation, please refer to the [Asking for Help](#asking-for-help) section to learn where to
+find help with your specific issue.
+
+## Issues when uploading datasets with `push_to_hub`
+
+### Authentication issues
+
+If you are experiencing authentication issues when sharing a dataset on 🤗 Hub using [`Dataset.push_to_hub`] and a Hugging Face 
+access token:
+
+* Make sure that the Hugging Face token you're using to authenticate yourself is a token with **write** permission.
+* On OSX, it may help to clean up all the huggingface.co passwords on your keychain access, as well as reconfigure `git config --global credential.helper osxkeychain`, before using `huggingface-cli login`.
+
+Alternatively, you can use SSH keys to authenticate yourself - read more in the [🤗 Hub documentation](https://huggingface.co/docs/hub/security-git-ssh).
+ 
+### Lost connection on large dataset upload 
+
+When uploading large datasets to Hub, if the number of dataset shards is large, it can create too many commits for the Hub in a 
+short period. This will result in a connection error.
+The connection error can also be caused by a HTTP 500 error returned by AWS S3 bucket that Hub uses internally.
+In either situation, you can re-run [`Dataset.push_to_hub`] to proceed with the dataset upload. Hub will check the SHAs 
+of already uploaded shards to avoid reuploading them. 
+We are working on making upload process more robust to transient errors, so updating to the latest library version is 
+always a good idea.
+
+### `Too Many Requests`
+
+Uploading large datasets via `push_to_hub()` can result in an error: 
+
+```bash
+HfHubHTTPError: 429 Client Error: Too Many Requests for url: ...
+You have exceeded our hourly quotas for action: commit. We invite you to retry later.
+```
+
+If you encounter this issue, you need to upgrade the `datasets` library to the latest version (or at least `2.15.0`).
+
+## Issues when creating datasets from custom data
+
+### Loading images and audio from a folder
+
+When creating a dataset from a folder, one of the most common issues is that the file structure does not follow the 
+expected format, or there's an issue with the metadata file.
+
+Learn more about required folder structure in corresponding documentation pages:
+
+* [AudioFolder](https://huggingface.co/docs/datasets/audio_dataset#audiofolder)
+* [ImageFolder](https://huggingface.co/docs/datasets/image_dataset#imagefolder)
+
+
+### Pickling issues 
+
+#### Pickling issues when using `Dataset.from_generator`
+
+When creating a dataset, [`IterableDataset.from_generator`] and [`Dataset.from_generator`] expect a "picklable" generator function.
+This is required to hash the function using [`pickle`](https://docs.python.org/3/library/pickle.html) to be able to cache the dataset on disk.
+
+While generator functions are generally "picklable", note that generator objects are not. So if you're using a generator object, 
+you will encounter a `TypeError` like this:
+
+```bash
+TypeError: cannot pickle 'generator' object
+```
+
+This error can also occur when using a generator function that uses a global object that is not "picklable", such as a 
+DB connection, for example. If that's the case, you can initialize such object directly inside the generator function to 
+avoid this error.
+
+#### Pickling issues with `Dataset.map`
+
+Pickling errors can also happen in the multiprocess [`Dataset.map`] - objects are pickled to be passed to child processes.
+If the objects used in the transformation are not picklable, it's not possible to cache the result of `map`, which leads to an error being raised. 
+
+Here are some ways to address this issue:
+* A universal solution to pickle issues is to make sure the objects (or generator classes) are pickable manually by implementing `__getstate__` / `__setstate__` / `__reduce__`.
+* You can also provide your own unique hash in `map` with the `new_fingerprint` argument.
+* You can also disable caching by calling `datasets.disable_caching()`, however, this is undesirable - [read more about importance of cache](cache)
+
+## Asking for help
+
+If the above troubleshooting advice did not help you resolve your issue, reach out for help to the community and the team.
+
+### Forums 
+
+Ask for help on the Hugging Face forums - post your question in the [🤗Datasets category](https://discuss.huggingface.co/c/datasets/10) 
+Make sure to write a descriptive post with relevant context about your setup and reproducible code to maximize the likelihood that your problem is solved!
+
+### Discord
+
+Post a question on [Discord](http://hf.co/join/discord), and let the team and the community help you.
+
+### Community Discussions on 🤗 Hub
+
+If you are facing issues creating a custom dataset on Hub, you can ask the Hugging Face team for help by opening a discussion in the Community tab of your dataset with this message:
+
+```text
+# Dataset rewiew request for <Dataset name>
+
+## Description
+
+<brief description of the dataset>
+
+## Files to review
+
+- file1
+- file2
+- ...
+
+cc @lhoestq @albertvillanova
+```
+
+### GitHub Issues
+
+Finally, if you suspect to have found a bug related to the library itself, create an Issue on the 🤗 Datasets 
+[GitHub repository](https://github.com/huggingface/datasets/issues). Include context regarding the bug: code snippet to reproduce,
+details about your environment and data, etc. to help us figure out what's wrong and how we can fix it.

docs/source/tsfile_load.mdx

@@ -0,0 +1,172 @@
+# Load TsFile data
+
+[TsFile](https://tsfile.apache.org/) is a columnar file format designed for time-series data and used as the native storage layer of [Apache IoTDB](https://iotdb.apache.org/). Compared with general-purpose columnar formats such as Parquet, TsFile is aware of the time-series data model (timestamps, devices, and measurements) and maintains an internal time index that enables time-range pruning without scanning entire files.
+
+This loader is provided as a separate guide because it does not follow the usual one-row-per-record tabular convention: each output row corresponds to one *device*, and per-measurement values are returned as Arrow `list<...>` columns. The mapping is described in detail below.
+
+## Installation
+
+The loader depends on the [`tsfile`](https://pypi.org/project/tsfile/) Python package:
+
+```bash
+pip install "tsfile>=2.3.0"
+```
+
+## Data model and output layout
+
+The loader follows the TsFile *table model*. Each table column is one of:
+
+- **TAG** — a string-typed identifier; the tuple of TAG values uniquely identifies a *device* (i.e. a single time-series source).
+- **FIELD** — a measurement whose value evolves over time.
+- **TIME** — the timestamp column, named `time` by default.
+
+The loader emits one dataset row per device. Within a row, the `time` column and every FIELD column are Arrow `list<...>` columns containing that device's full time series, sorted in ascending time order. TAG columns appear as scalar `string` columns.
+
+Concretely, the output schema has the form:
+
+```text
+<tag_1>:    string
+<tag_2>:    string                       # one column per TAG
+...
+time:       list<timestamp[unit, tz]>
+<field_1>:  list<original_type>          # one column per FIELD
+<field_2>:  list<original_type>
+...
+```
+
+When the same device appears in multiple input files of a split, its per-file chunks are concatenated and sorted by timestamp before being emitted as a single row. Duplicate timestamps for the same device raise `ValueError`.
+
+## Basic usage
+
+Load a single TsFile:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("tsfile", data_files="my_data.tsfile")
+```
+
+Map files to splits explicitly:
+
+```py
+>>> dataset = load_dataset(
+...     "tsfile",
+...     data_files={"train": "train_data.tsfile", "test": "test_data.tsfile"},
+... )
+```
+
+## Example dataset on the Hub
+
+A ready-to-use example is available at [`tsfile/lotsa_data`](https://huggingface.co/datasets/tsfile/lotsa_data). Because `.tsfile` files are recognized automatically, you can load it by repository id without specifying `data_files`:
+
+```py
+>>> from datasets import load_dataset
+>>> dataset = load_dataset("tsfile/lotsa_data")
+>>> dataset
+DatasetDict({
+    train: Dataset({
+        features: ['timeseries_id', 'time', 'value'],
+        num_rows: 91
+    })
+})
+```
+
+Each row is one device. The TAG column `timeseries_id` identifies the device, while `time` and `value` are `list<...>` columns holding that device's full series:
+
+```py
+>>> row = dataset["train"][0]
+>>> row["timeseries_id"]
+'Bear_assembly_Angel'
+>>> len(row["time"]), len(row["value"])
+(8760, 8760)
+>>> row["time"][:3]
+[datetime.datetime(2017, 1, 1, 0, 0), datetime.datetime(2017, 1, 1, 1, 0), datetime.datetime(2017, 1, 1, 2, 0)]
+```
+
+## Selecting a table
+
+A TsFile can contain multiple tables. When `table_name` is omitted, the first table found in the first valid file is used. Lookups are case-insensitive.
+
+```py
+>>> dataset = load_dataset("tsfile", data_files="my_data.tsfile", table_name="sensor_data")
+```
+
+## Selecting columns
+
+`columns` restricts the FIELD columns that are read. The TAG columns and the `time` column are always returned because they identify the device and its timeline. Names in `columns` that refer to a TAG or to the `time` column are silently ignored (they are emitted as usual, just once); names that match a field absent from every file become all-null list columns.
+
+```py
+>>> dataset = load_dataset(
+...     "tsfile",
+...     data_files="my_data.tsfile",
+...     columns=["temperature", "humidity"],
+... )
+```
+
+## Filtering by time range
+
+`start_time` and `end_time` are inclusive bounds; either may be omitted. The bounds are pushed down to TsFile's internal time index, so only the matching data blocks are read from disk. Both bounds accept any of:
+
+- `int` — raw epoch in `timestamp_unit` (default milliseconds);
+- `datetime.datetime` — naive values are interpreted as UTC, tz-aware values are converted to UTC;
+- `datetime.date`;
+- ISO-8601 `str`, e.g. `"2024-01-01T00:00:00"`;
+- `pyarrow.TimestampScalar`.
+
+```py
+>>> from datetime import datetime
+>>> dataset = load_dataset(
+...     "tsfile",
+...     data_files="my_data.tsfile",
+...     start_time=datetime(2023, 11, 14),
+...     end_time="2023-11-15T00:00:00",
+... )
+```
+
+## Schema evolution across files
+
+When different files expose different columns — for example a new sensor field is introduced later — the loader takes the union of all FIELD columns and fills missing values with nulls. Numeric FIELD types are promoted following IoTDB's widening rules (`INT32 → INT64 → DOUBLE`, `INT32 → FLOAT → DOUBLE`).
+
+```py
+>>> dataset = load_dataset("tsfile", data_files=["day1.tsfile", "day2.tsfile"])
+```
+
+## Handling unreadable files
+
+By default, an unreadable or non-TsFile input raises an error. Set `on_bad_files` to `"warn"` to log and continue, or `"skip"` to silently drop the file.
+
+```py
+>>> dataset = load_dataset("tsfile", data_files="data/*.tsfile", on_bad_files="skip")
+```
+
+## Timestamp unit and time zone
+
+`timestamp_unit` (default `"ms"`, matching IoTDB) controls the resolution of the `time` column and the interpretation of integer time bounds. `timestamp_tz` attaches a time zone to the Arrow timestamp type; `None` (the default) yields a timezone-naive type.
+
+```py
+>>> dataset = load_dataset(
+...     "tsfile",
+...     data_files="my_data.tsfile",
+...     timestamp_unit="us",
+...     timestamp_tz="UTC",
+... )
+```
+
+## Memory and batching
+
+Two parameters control memory usage:
+
+- `input_batch_size` (default `65_536`) — maximum number of rows fetched per Arrow batch from `TsFileReader.query_table`. Bounds peak memory while streaming a single device.
+- `output_batch_size` (default `32`) — number of devices packed into each Arrow record batch yielded to the writer. Smaller values give more responsive progress reporting; larger values reduce per-batch overhead.
+
+```py
+>>> dataset = load_dataset(
+...     "tsfile",
+...     data_files="large_data.tsfile",
+...     input_batch_size=32_768,
+...     output_batch_size=128,
+... )
+```
+
+Peak memory is bounded by the payload of a single device across the split, not by the size of the split as a whole.
+
+See [`~datasets.packaged_modules.tsfile.TsFileConfig`] for the full list of parameters.

docs/source/tutorial.md

@@ -0,0 +1,12 @@
+# Overview
+
+Welcome to the 🤗 Datasets tutorials! These beginner-friendly tutorials will guide you through the fundamentals of working with 🤗 Datasets. You'll load and prepare a dataset for training with your machine learning framework of choice. Along the way, you'll learn how to load different dataset configurations and splits, interact with and see what's inside your dataset, preprocess, and share a dataset to the [Hub](https://huggingface.co/datasets).
+
+The tutorials assume some basic knowledge of Python and a machine learning framework like PyTorch or TensorFlow. If you're already familiar with these, feel free to check out the [quickstart](./quickstart) to see what you can do with 🤗 Datasets.
+
+> [!TIP]
+> The tutorials only cover the basic skills you need to use 🤗 Datasets. There are many other useful functionalities and applications that aren't discussed here. If you're interested in learning more, take a look at [Chapter 5](https://huggingface.co/course/chapter5/1?fw=pt) of the Hugging Face course.
+
+If you have any questions about 🤗 Datasets, feel free to join and ask the community on our [forum](https://discuss.huggingface.co/c/datasets/10).
+
+Let's get started! 🏁