chore: import upstream snapshot with attribution

.github/workflows/cffconvert.yml

@@ -0,0 +1,23 @@
+name: cffconvert
+
+on:
+  push:
+    paths:
+      - CITATION.cff
+  workflow_dispatch:
+
+permissions:
+  contents: read
+
+jobs:
+  validate:
+    name: "validate"
+    runs-on: ubuntu-latest
+    steps:
+      - name: Check out a copy of the repository
+        uses: actions/checkout@v7
+
+      - name: Check whether the citation metadata from CITATION.cff is valid
+        uses: citation-file-format/cffconvert-github-action@2.0.0
+        with:
+          args: "--validate"

.github/workflows/ci.yml

@@ -0,0 +1,160 @@
+name: ci-workflow
+
+on: [push, pull_request, workflow_dispatch]
+
+permissions:
+  contents: read
+
+env:
+  THIRD_PARTY_DIR: ${{ github.workspace }}/third
+  CORENLP: ${{ github.workspace }}/third/stanford-corenlp
+  CORENLP_MODELS: ${{ github.workspace }}/third/stanford-corenlp
+  STANFORD_PARSER: ${{ github.workspace }}/third/stanford-parser
+  STANFORD_MODELS: ${{ github.workspace }}/third/stanford-postagger
+  STANFORD_POSTAGGER: ${{ github.workspace }}/third/stanford-postagger
+  SENNA: ${{ github.workspace }}/third/senna
+  PROVER9: ${{ github.workspace }}/third/prover9/bin
+  MEGAM: ${{ github.workspace }}/third/megam
+  MALT_PARSER: ${{ github.workspace }}/third/maltparser
+
+jobs:
+  pre-commit:
+    name: pre-commit
+    runs-on: ubuntu-latest
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.13"  # or your chosen version
+
+      - name: Install pre-commit
+        run: pip install pre-commit
+
+      - name: Run pre-commit hooks
+        run: pre-commit run --all-files
+
+  minimal_download_test:
+    name: Minimal NLTK Download Test
+    runs-on: ${{ matrix.os }}
+    strategy:
+      matrix:
+        os: [ubuntu-latest, macos-latest, windows-latest]
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: "3.14"
+
+      - name: Install core dependencies
+        run: pip install regex defusedxml
+
+      - name: Set NLTK_DATA environment variable
+        shell: bash
+        run: echo "NLTK_DATA=${{ github.workspace }}/nltk_data" >> $GITHUB_ENV
+
+      - name: Show NLTK_DATA in shell
+        shell: bash
+        run: |
+          echo "NLTK_DATA in shell: $NLTK_DATA"
+
+      - name: Ensure minimal NLTK data for cache
+        shell: bash
+        run: |
+          python -c "import os, nltk; d = os.environ['NLTK_DATA']; import pathlib; pathlib.Path(d).mkdir(parents=True, exist_ok=True); nltk.download('wordnet', download_dir=d)"
+
+  test:
+    name: Python ${{ matrix.python-version }} on ${{ matrix.os }}
+    needs: [pre-commit, minimal_download_test]
+    strategy:
+      matrix:
+        python-version: ['3.10', '3.11', '3.12', '3.13', '3.14', '3.14t']
+        os: [ubuntu-latest, macos-latest, windows-latest]
+        exclude:
+          - os: windows-latest
+            python-version: '3.14t'  # scikit-learn issue on Py3.14t on Windows
+      fail-fast: false
+    runs-on: ${{ matrix.os }}
+    steps:
+      - uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: ${{ matrix.python-version }}
+
+      - name: Set NLTK_DATA environment variable
+        shell: bash
+        run: echo "NLTK_DATA=${{ github.workspace }}/nltk_data" >> $GITHUB_ENV
+
+      - name: Install dependencies
+        run: |
+          pip install --upgrade pip
+          pip install --upgrade --requirement requirements-ci.txt
+
+      - name: Ensure minimal NLTK data for cache
+        shell: bash
+        run: |
+          python -c "import os, nltk; d = os.environ['NLTK_DATA']; import pathlib; pathlib.Path(d).mkdir(parents=True, exist_ok=True); nltk.download('wordnet', download_dir=d)"
+
+      - name: Show NLTK_DATA and workspace
+        shell: bash
+        run: |
+          echo "GITHUB_WORKSPACE is: $GITHUB_WORKSPACE"
+          echo "NLTK_DATA is: $NLTK_DATA"
+          python -c "import os; print('Python sees GITHUB_WORKSPACE:', os.environ.get('GITHUB_WORKSPACE')); print('Python sees NLTK_DATA:', os.environ.get('NLTK_DATA'))"
+
+      - name: List contents of NLTK data dir
+        shell: bash
+        run: ls -lR "${{ github.workspace }}/nltk_data" || echo "nltk_data not found"
+
+      - name: Cache nltk data
+        uses: actions/cache@v6
+        id: nltk-data-cache
+        with:
+          path: ${{ github.workspace }}/nltk_data
+          key: nltk_data_${{ runner.os }}_v2
+
+      - name: Download nltk data on cache miss
+        if: steps.nltk-data-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          python -c "import os; import nltk; from pathlib import Path; path = Path(os.environ['NLTK_DATA']); path.mkdir(parents=True, exist_ok=True); nltk.download('all', download_dir=path)"
+
+      # --- THIRD PARTY TOOLS CACHE SECTION ---
+      - name: Ensure third-party directory exists
+        run: mkdir -p "${{ env.THIRD_PARTY_DIR }}"
+
+      - name: Cache third-party tools
+        uses: actions/cache@v6
+        id: third-party-cache
+        with:
+          path: ${{ env.THIRD_PARTY_DIR }}
+          key: third_${{ runner.os }}_${{ hashFiles('tools/github_actions/third-party.sh') }}_v1
+
+      - name: List contents of third-party dir before download
+        shell: bash
+        run: ls -lR "${{ env.THIRD_PARTY_DIR }}" || echo "third-party dir not found"
+
+      - name: Download third-party data on cache miss
+        if: steps.third-party-cache.outputs.cache-hit != 'true'
+        shell: bash
+        run: |
+          chmod +x ./tools/github_actions/third-party.sh
+          ./tools/github_actions/third-party.sh
+
+      - name: List contents of third-party dir after download/cache
+        shell: bash
+        run: ls -lR "${{ env.THIRD_PARTY_DIR }}" || echo "third-party dir not found"
+
+      - name: Print NLTK data search paths
+        shell: bash
+        run: python -c "import nltk; print('NLTK data search paths:', nltk.data.path)"
+
+      - name: Run pytest
+        shell: bash
+        run: |
+          pytest --numprocesses auto -rsx --doctest-modules nltk

.github/workflows/labeler.yml

@@ -0,0 +1,15 @@
+name: "Pull Request Labeler"
+on:
+- pull_request_target
+
+jobs:
+  triage:
+    permissions:
+      contents: read
+      pull-requests: write
+    runs-on: ubuntu-latest
+    steps:
+    - uses: actions/labeler@v6
+      with:
+        repo-token: "${{ secrets.GITHUB_TOKEN }}"
+        sync-labels: true

.github/workflows/release.yml

@@ -0,0 +1,112 @@
+name: Release
+
+on:
+  push:
+    tags:
+      - 'v*'
+
+jobs:
+  gate-on-ci:
+    name: Gate release on CI success for this tag commit
+    runs-on: ubuntu-latest
+    permissions:
+      actions: read
+      contents: read
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+        with:
+          fetch-depth: 0
+
+      - name: Check that release tag points to a develop commit
+        run: |
+          git merge-base --is-ancestor "$GITHUB_SHA" origin/develop
+
+      - name: Verify version string in documentation
+        run: |
+          # Strip leading 'v'
+          RAW_VERSION=${GITHUB_REF_NAME#v}
+          # Strip pre-release suffixes (e.g., -rc1, -beta)
+          VERSION=${RAW_VERSION%%-*}
+          echo "Checking files for base version: $VERSION"
+
+          for file in nltk/VERSION ChangeLog web/news.rst web/conf.py; do
+            if grep -Fq "$VERSION" "$file"; then
+              echo "✅ Found $VERSION in $file"
+            else
+              echo "❌ ERROR: Version $VERSION not found in $file!"
+              exit 1
+            fi
+          done
+
+      - name: Wait for ci.yml to complete and verify success
+        env:
+          GH_TOKEN: ${{ secrets.GITHUB_TOKEN }}
+        run: |
+          set -euo pipefail
+
+          workflow_file="ci.yml"
+          attempts=60
+          delay=20
+
+          for _ in $(seq 1 "$attempts"); do
+            run=$(gh api \
+              "repos/${{ github.repository }}/actions/workflows/${workflow_file}/runs?per_page=100" \
+              --jq '[.workflow_runs[] | select(.head_sha == "'"$GITHUB_SHA"'")] | sort_by(.run_number) | last')
+
+            if [ -z "$run" ] || [ "$run" = "null" ]; then
+              sleep "$delay"
+              continue
+            fi
+
+            status=$(jq -r '.status // empty' <<< "$run")
+            conclusion=$(jq -r '.conclusion // empty' <<< "$run")
+
+            if [ "$status" != "completed" ]; then
+              sleep "$delay"
+              continue
+            fi
+
+            if [ "$conclusion" = "success" ]; then
+              exit 0
+            fi
+
+            run_id=$(jq -r '.id // empty' <<< "$run")
+            run_number=$(jq -r '.run_number // empty' <<< "$run")
+            run_url="https://github.com/${{ github.repository }}/actions/runs/${run_id}"
+            echo "CI failed with conclusion: ${conclusion:-unknown} (run #${run_number:-unknown}). See workflow run: ${run_url}"
+            exit 1
+          done
+
+          echo "Timed out waiting for CI to complete for $GITHUB_SHA"
+          exit 1
+
+  build-and-github-release:
+    needs: gate-on-ci
+    name: Build and create GitHub draft release
+    runs-on: ubuntu-latest
+    permissions:
+      contents: write
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v7
+
+      - name: Set up Python
+        uses: actions/setup-python@v6
+        with:
+          python-version: '3.12'
+
+      - name: Install build dependencies
+        run: |
+          python -m pip install --upgrade pip
+          pip install build
+
+      - name: Build package
+        run: python -m build
+
+      - name: Create GitHub Release
+        uses: softprops/action-gh-release@v3
+        with:
+          files: dist/*
+          draft: true
+          generate_release_notes: true

.github/workflows/retest-pr.yml

@@ -0,0 +1,71 @@
+name: Retest PR
+
+on:
+  issue_comment:
+    types: [created]
+
+env:
+  FORCE_JAVASCRIPT_ACTIONS_TO_NODE24: true
+
+jobs:
+  retest:
+    if: github.event.issue.pull_request && contains(github.event.comment.body, 'retest')
+    runs-on: ubuntu-latest
+    permissions:
+      actions: write
+      issues: write
+      pull-requests: read
+    steps:
+      - name: Check authorization and trigger phrase
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const assoc = context.payload.comment.author_association;
+            if (!['OWNER', 'MEMBER', 'COLLABORATOR'].includes(assoc)) {
+              core.setFailed('Not authorized (association: ' + assoc + ')');
+              return;
+            }
+            const body = context.payload.comment.body;
+            if (!body.includes('/retest') &&
+                !body.includes('[CI: retest]') &&
+                !body.includes('[ci: retest]')) {
+              core.setFailed('No recognized retest trigger found.');
+            }
+
+      - name: Get PR head SHA
+        id: pr
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const pr = (await github.rest.pulls.get({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              pull_number: context.issue.number,
+            })).data;
+            core.setOutput('sha', pr.head.sha);
+
+      - name: Re-run CI
+        uses: actions/github-script@v9
+        with:
+          script: |
+            const runs = await github.rest.actions.listWorkflowRuns({
+              owner: context.repo.owner,
+              repo: context.repo.repo,
+              workflow_id: 'ci.yml',
+              head_sha: '${{ steps.pr.outputs.sha }}',
+              per_page: 1,
+            });
+            if (runs.data.workflow_runs.length === 0) {
+              core.setFailed('No CI run found for this PR head SHA.');
+              return;
+            }
+            const run = runs.data.workflow_runs[0];
+            if (run.status === 'completed') {
+              await github.rest.actions.reRunWorkflow({
+                owner: context.repo.owner,
+                repo: context.repo.repo,
+                run_id: run.id,
+              });
+            } else {
+              core.info('CI is already running (status: ' + run.status + '). Nothing to re-run.');
+            }