chore: import upstream snapshot with attribution

.github/ISSUE_TEMPLATE/bug-report.yml

@@ -0,0 +1,45 @@
+name: Bug report
+description: Create a report to help reproduce and fix the bug
+body:
+  - type: textarea
+    id: description
+    attributes:
+      label: Describe the bug
+      description: A clear and concise description of what the bug is
+    validations:
+      required: true
+  
+  - type: textarea
+    id: reproduction
+    attributes:
+      label: Steps to reproduce the bug
+      description: |
+        Please provide a code sample that reproduces the problem you ran into. It can be a Colab link or just a code snippet.
+        If you have code snippets, error messages, stack traces please provide them here as well.
+        Important! Use code tags to correctly format your code. See https://help.github.com/en/github/writing-on-github/creating-and-highlighting-code-blocks#syntax-highlighting
+        Do not use screenshots, as they are hard to read and (more importantly) don't allow others to copy-and-paste your code.
+      placeholder: |
+        Steps to reproduce the behavior:
+          
+          1.
+          2.
+          3.
+    validations:
+      required: true
+
+  - type: textarea
+    id: expected-behavior
+    validations:
+      required: true
+    attributes:
+      label: Expected behavior
+      description: A clear and concise description of the expected results.
+
+  - type: textarea
+    id: environment-info
+    attributes:
+      label: Environment info
+      description: Please share your environemnt info with us. You can run the command `datasets-cli env` and copy-paste its output below.
+      placeholder: datasets version, platform, python version, ...
+    validations:
+      required: true

.github/ISSUE_TEMPLATE/config.yml

@@ -0,0 +1,7 @@
+contact_links:
+  - name: Datasets on the Hugging Face Hub
+    url: https://huggingface.co/datasets
+    about: Please use the "Community" tab of the dataset on the Hugging Face Hub to open a discussion or a pull request
+  - name: Forum
+    url: https://discuss.huggingface.co/c/datasets/10
+    about: Please ask and answer questions here, and engage with other community members

.github/ISSUE_TEMPLATE/feature-request.yml

@@ -0,0 +1,29 @@
+name: Feature request
+description: Suggest an idea for this project
+labels: ["enhancement"]
+body:
+  - type: textarea
+    id: feature-request
+    attributes:
+      label: Feature request
+      description: A clear and concise description of the feature proposal.
+    validations:
+      required: true
+  
+  - type: textarea
+    id: motivation
+    validations:
+      required: true
+    attributes:
+      label: Motivation
+      description: |
+        Please outline the motivation for the proposal. Is your feature request related to a problem? e.g., I'm always frustrated when [...]. If this is related to another GitHub issue, please link here too.   
+
+  - type: textarea
+    id: contribution
+    validations:
+      required: true
+    attributes:
+      label: Your contribution
+      description: |
+        Is there any way that you could help, e.g. by submitting a PR? Make sure to read the CONTRIBUTING.MD [readme](https://github.com/huggingface/datasets/blob/main/CONTRIBUTING.md).

.github/conda/build.sh

@@ -0,0 +1 @@
+$PYTHON setup.py install --single-version-externally-managed --record=record.txt

.github/conda/meta.yaml

@@ -0,0 +1,55 @@
+{% set name = "datasets" %}
+
+package:
+  name: "{{ name|lower }}"
+  version: "{{ DATASETS_VERSION }}"
+
+source:
+  path: ../../
+
+build:
+  noarch: python
+
+requirements:
+  host:
+    - python
+    - pip
+    - numpy >=1.17
+    - pyarrow >=16.0.0
+    - python-xxhash
+    - dill
+    - pandas
+    - requests >=2.19.0
+    - httpx <1.0.0
+    - tqdm >=4.66.3
+    - dataclasses
+    - multiprocess
+    - fsspec
+    - huggingface_hub >=0.25.0,<2.0.0
+    - packaging
+  run:
+    - python
+    - pip
+    - numpy >=1.17
+    - pyarrow >=16.0.0
+    - python-xxhash
+    - dill
+    - pandas
+    - requests >=2.19.0
+    - httpx <1.0.0
+    - tqdm >=4.66.3
+    - dataclasses
+    - multiprocess
+    - fsspec
+    - huggingface_hub >=0.25.0,<2.0.0
+    - packaging
+
+test:
+  imports:
+    - datasets
+
+about:
+  home: https://huggingface.co
+  license: Apache License 2.0
+  license_file: LICENSE
+  summary: "🤗 The largest hub of ready-to-use NLP datasets for ML models with fast, easy-to-use and efficient data manipulation tools"

.github/workflows/build_documentation.yml

@@ -0,0 +1,20 @@
+name: Build documentation
+
+on:
+  push:
+    branches:
+      - main
+      - doc-builder*
+      - v*-release
+      - v*-patch
+
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/build_main_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
+    with:
+      commit_sha: ${{ github.sha }}
+      package: datasets
+      notebook_folder: datasets_doc
+    secrets:
+      token: ${{ secrets.HUGGINGFACE_PUSH }}
+      hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}

.github/workflows/build_pr_documentation.yml

@@ -0,0 +1,16 @@
+name: Build PR Documentation
+
+on:
+  pull_request:
+
+concurrency:
+  group: ${{ github.workflow }}-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/build_pr_documentation.yml@90b4ee2c10b81b5c1a6367c4e6fc9e2fb510a7e3  # main
+    with:
+      commit_sha: ${{ github.event.pull_request.head.sha }}
+      pr_number: ${{ github.event.number }}
+      package: datasets

.github/workflows/ci.yml

@@ -0,0 +1,176 @@
+name: CI
+
+on:
+  pull_request:
+    branches:
+      - main
+  push:
+    branches:
+      - main
+      - ci-*
+
+env:
+  CI_HEADERS: ${{ secrets.CI_HEADERS }}
+
+jobs:
+
+  check_code_quality:
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      - name: Set up Python
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.10"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install .[quality]
+      - name: Check quality
+        run: |
+          ruff check tests src benchmarks utils setup.py # linter
+          ruff format --check tests src benchmarks utils setup.py # formatter
+
+  test:
+    needs: check_code_quality
+    strategy:
+      matrix:
+        test: ['unit', 'integration']
+        os: [ubuntu-latest, windows-latest]
+        deps_versions: [deps-latest, deps-minimum]
+    continue-on-error: ${{ matrix.test == 'integration' }}
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+      - name: Setup FFmpeg
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg 
+      - name: Set up Python 3.10
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.10"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.10"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Install uv
+        run: pip install --upgrade uv
+      - name: Install dependencies
+        run: uv pip install --system "datasets[tests] @ ."
+      - name: Install dependencies (latest versions)
+        if: ${{ matrix.deps_versions == 'deps-latest' }}
+        run: uv pip install --system --upgrade pyarrow huggingface-hub "dill<0.3.9"
+      - name: Install dependencies (minimum versions)
+        if: ${{ matrix.deps_versions != 'deps-latest' }}
+        run: uv pip install --system pyarrow==21.0.0 huggingface-hub==0.25.0 transformers dill==0.3.1.1
+      - name: Print dependencies
+        run: uv pip list
+      - name: Test with pytest
+        run: |
+          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/
+
+  test_py314:
+    needs: check_code_quality
+    strategy:
+      matrix:
+        test: ['unit']
+        os: [ubuntu-latest, windows-latest]
+        deps_versions: [deps-latest]
+    continue-on-error: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+      - name: Setup FFmpeg
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.14"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.14"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Install uv
+        run: pip install --upgrade uv
+      - name: Install dependencies
+        run: uv pip install --system "datasets[tests] @ ."
+      - name: Install tsfile (py3.14 only)
+        run: uv pip install --system "tsfile>=2.3.0"
+      - name: Print dependencies
+        run: uv pip list
+      - name: Test with pytest
+        run: |
+          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/
+
+  test_py314_future:
+    needs: check_code_quality
+    strategy:
+      matrix:
+        test: ['unit']
+        os: [ubuntu-latest, windows-latest]
+        deps_versions: [deps-latest]
+    continue-on-error: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+        with:
+          fetch-depth: 0
+      - name: Setup FFmpeg
+        if: ${{ matrix.os == 'ubuntu-latest' }}
+        run: |
+          sudo apt update
+          sudo apt install -y ffmpeg 
+      - name: Set up Python 3.14
+        uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065  # v5
+        with:
+          python-version: "3.14"
+      - name: Setup conda env (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        uses: conda-incubator/setup-miniconda@9f54435e0e72c53962ee863144e47a4b094bfd35  # v2
+        with:
+          auto-update-conda: true
+          miniconda-version: "latest"
+          activate-environment: test
+          python-version: "3.14"
+      - name: Setup FFmpeg (windows)
+        if: ${{ matrix.os == 'windows-latest' }}
+        run: conda install "ffmpeg=7.0.1" -c conda-forge
+      - name: Upgrade pip
+        run: python -m pip install --upgrade pip
+      - name: Install uv
+        run: pip install --upgrade uv
+      - name: Install dependencies
+        run: uv pip install --system "datasets[tests_numpy2] @ ."
+      - name: Print dependencies
+        run: pip list
+
+      - name: Test with pytest
+        run: |
+          python -m pytest -rfExX -m ${{ matrix.test }} -n 2 --dist loadfile -sv ./tests/

.github/workflows/release-conda.yml

@@ -0,0 +1,47 @@
+name: Release - Conda
+
+on:
+  push:
+    tags:
+      - "[0-9]+.[0-9]+.[0-9]+*"
+
+env:
+  ANACONDA_API_TOKEN: ${{ secrets.ANACONDA_API_TOKEN }}
+
+jobs:
+  build_and_package:
+    runs-on: ubuntu-22.04
+    defaults:
+      run:
+        shell: bash -l {0}
+
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+
+      - name: Install miniconda
+        uses: conda-incubator/setup-miniconda@8ee1f361103df19b6f8c8655fd3967a8ecb162d5  # v4.0.1
+        with:
+          auto-update-conda: true
+          auto-activate-base: false
+          python-version: "3.10"
+          channels: huggingface
+
+      - name: Setup conda env
+        run: |
+          # Install conda-build into base so the command is available
+          conda install -y -c defaults -c conda-forge anaconda-client conda-build
+          # Create the build environment
+          conda create -y -n build-datasets python=3.10
+          conda activate build-datasets
+
+      - name: Extract version
+        run: echo "DATASETS_VERSION=`python setup.py --version`" >> $GITHUB_ENV
+
+      - name: Build conda packages
+        run: |
+          conda-build --override-channels -c conda-forge -c huggingface .github/conda
+
+      - name: Upload to Anaconda
+        run: |
+          anaconda upload `conda-build .github/conda --output -c conda-forge` --force

.github/workflows/self-assign.yaml

@@ -0,0 +1,16 @@
+name: Self-assign
+on:
+  issue_comment:
+    types: created
+jobs:
+  one:
+    runs-on: ubuntu-latest
+    if: >-
+      (github.event.comment.body == '#take' ||
+       github.event.comment.body == '#self-assign')
+      && !github.event.issue.assignee
+    steps:
+      - run: |
+          echo "Assigning issue ${{ github.event.issue.number }} to ${{ github.event.comment.user.login }}"
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -d '{"assignees": ["${{ github.event.comment.user.login }}"]}' https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/assignees
+          curl -H "Authorization: token ${{ secrets.GITHUB_TOKEN }}" -X "DELETE" https://api.github.com/repos/${{ github.repository }}/issues/${{ github.event.issue.number }}/labels/help%20wanted

.github/workflows/trufflehog.yml

@@ -0,0 +1,20 @@
+on:
+  push:
+
+name: Secret Leaks
+
+permissions:
+  contents: read
+
+jobs:
+  trufflehog:
+    runs-on: ubuntu-latest
+    steps:
+    - name: Checkout code
+      uses: actions/checkout@de0fac2e4500dabe0009e67214ff5f5447ce83dd  # v6.0.2
+      with:
+        fetch-depth: 0
+    - name: Secret Scanning
+      uses: trufflesecurity/trufflehog@6bd2d14f7a4bc1e569fa3550efa7ec632a4fa67b  # main
+      with:
+        extra_args: --results=verified

.github/workflows/upload_pr_documentation.yml

@@ -0,0 +1,16 @@
+name: Upload PR Documentation
+
+on:
+  workflow_run:
+    workflows: ["Build PR Documentation"]
+    types:
+      - completed
+
+jobs:
+  build:
+    uses: huggingface/doc-builder/.github/workflows/upload_pr_documentation.yml@9ad2de8582b56c017cb530c1165116d40433f1c6  # main
+    with:
+      package_name: datasets
+    secrets:
+      hf_token: ${{ secrets.HF_DOC_BUILD_PUSH }}
+      comment_bot_token: ${{ secrets.COMMENT_BOT_TOKEN }}