nltk--nltk/nltk/test/unit/test_data_security.py

import gzip
import os
import zipfile

import pytest

import nltk.data as data
from nltk import pathsec


def test_normalize_rejects_no_protocol_traversal():
    """No-protocol traversal sequences should be rejected."""
    with pytest.raises(ValueError):
        data.normalize_resource_url("../../etc/passwd")

    with pytest.raises(ValueError):
        data.normalize_resource_url("../relative/../etc/passwd")


def test_normalize_rejects_no_protocol_backslashes():
    """Windows-style backslash traversal should be rejected when no protocol is present."""
    with pytest.raises(ValueError):
        data.normalize_resource_url(r"..\..\etc\passwd")


def test_normalize_allows_package_paths():
    """Valid package-style resource names should still be treated as nltk: URLs."""
    out = data.normalize_resource_url("corpora/brown")
    assert out.startswith(
        "nltk:"
    ), "Package-style paths should be treated as 'nltk:' URLs"


def test_find_rejects_traversal_direct_call():
    """Defense-in-depth: direct calls to find() should reject traversal-like names."""
    with pytest.raises(ValueError):
        data.find("../../etc/passwd")


def test_find_rejects_traversal_that_becomes_unsafe_after_normalization():
    """
    Defense-in-depth edge case: a path can become unsafe only after normalization.

    Example from review: "foo/../../etc/passwd" normalizes to "../etc/passwd" and
    must still be rejected.
    """
    with pytest.raises(ValueError):
        data.find("foo/../../etc/passwd")


def test_normalize_rejects_no_protocol_absolute_posix_path():
    """Absolute POSIX paths without a protocol should be rejected."""
    with pytest.raises(ValueError):
        data.normalize_resource_url("/etc/passwd")


def test_normalize_rejects_no_protocol_windows_drive_letter_paths():
    """
    Windows drive letter paths should be rejected even on non-Windows platforms.

    Review note: don't gate 'C:/etc/passwd' on Windows only; ensure robust rejection
    regardless of runtime platform.
    """
    with pytest.raises(ValueError):
        data.normalize_resource_url(r"C:\etc\passwd")

    # Run on all platforms (per review suggestion)
    with pytest.raises(ValueError):
        data.normalize_resource_url("C:/etc/passwd")


def test_normalize_rejects_no_protocol_dotdot_only():
    """A resource name that is exactly '..' should be rejected."""
    with pytest.raises(ValueError):
        data.normalize_resource_url("..")


@pytest.mark.parametrize(
    "url",
    [
        # encoded absolute path
        "nltk:%2fetc%2fpasswd",
        "nltk:%2Fetc%2Fpasswd",
        # encoded ".." traversal
        "nltk:corpora/%2e%2e/%2e%2e/etc/passwd",
        "nltk:corpora/%2E%2E/%2E%2E/etc/passwd",
        # encoded separators sandwiching literal ".."
        "nltk:corpora/..%2f..%2fetc%2fpasswd",
        # encoded /proc target
        "nltk:%2fproc%2fself%2fenviron",
        # encoded Windows drive letter
        "nltk:%43%3a%5cetc",
        # encoded backslash traversal
        "nltk:%5c..%5cetc%5cpasswd",
    ],
)
def test_normalize_rejects_url_encoded_traversal(url):
    """URL-encoded path separators and traversal must not bypass the safety check.

    Regression: prior to the fix, ``nltk.data.load("nltk:%2fetc%2fpasswd")``
    decoded the path inside ``url2pathname()`` *after* the safety check ran,
    allowing arbitrary file read. See huntr report
    https://huntr.com/bounties/fae662d6-74c2-44fa-95f3-f53d4e8a8355.
    """
    with pytest.raises(ValueError):
        data.normalize_resource_url(url)


@pytest.mark.parametrize(
    "name",
    [
        "%2fetc%2fpasswd",
        "corpora/%2e%2e/%2e%2e/etc/passwd",
        "corpora/..%2f..%2fetc%2fpasswd",
    ],
)
def test_find_rejects_url_encoded_traversal(name):
    """Defense-in-depth: find() must reject URL-encoded traversal directly."""
    with pytest.raises(ValueError):
        data.find(name)


@pytest.mark.parametrize(
    "url",
    [
        # Encoded space — extremely common in real resource names.
        "nltk:corpora/foo%20bar",
        # UTF-8-encoded non-ASCII name (here "中文").
        "nltk:corpora/%E4%B8%AD%E6%96%87",
        # Encoded ASCII dot in an extension — decodes to ``file.zip``, no traversal.
        "nltk:corpora/file%2Ezip",
        # Mixed safe encoding inside a realistic zipfile-style name.
        "nltk:tokenizers/punkt%2Ezip/punkt/PY3/english.pickle",
    ],
)
def test_normalize_allows_safe_percent_encoded_names(url):
    """Percent-encoded characters that decode to *safe* path components
    must not be falsely rejected.

    Guards against the centralised encoded-bypass check accidentally
    being tightened into a blanket "any percent-encoding is unsafe" rule.
    """
    out = data.normalize_resource_url(url)
    assert out.startswith("nltk:"), (
        f"Safe encoded name {url!r} should still normalise to an nltk: URL, "
        f"got {out!r}"
    )


@pytest.mark.parametrize(
    "url",
    [
        # Double-encoded "/" — single unquote yields literal "%2f", which is
        # neither a path separator nor a traversal segment.
        "nltk:%252fetc%252fpasswd",
        # Double-encoded ".." — single unquote yields literal "%2e%2e".
        "nltk:corpora/%252e%252e/etc/passwd",
    ],
)
def test_double_encoded_payloads_are_not_exploitable(url):
    """Double-encoded payloads must not reach the host filesystem.

    ``url2pathname()`` only performs one decoding pass, so a double-encoded
    payload resolves to a literal ``%2f...`` resource name *inside* the
    nltk_data search path rather than to ``/etc/passwd`` on the host. We
    intentionally do not chase the asymmetric depth of decoding here — we
    just assert the call fails closed (LookupError for a non-existent
    nltk resource, or ValueError if a future tightening rejects it) and
    never silently returns host-file contents.
    """
    with pytest.raises((LookupError, ValueError)):
        data.load(url, format="raw")


@pytest.mark.parametrize(
    "url",
    [
        "nltk:%2fetc%2fpasswd",
        "nltk:corpora/..%2f..%2fetc%2fpasswd",
        "nltk:corpora/%2e%2e/%2e%2e/etc/passwd",
    ],
)
def test_load_rejects_encoded_traversal_end_to_end(url):
    """End-to-end: the public ``data.load`` entry point must reject the
    same encoded-traversal payloads, not just the internal
    ``normalize_resource_url`` / ``find`` helpers.
    """
    with pytest.raises(ValueError):
        data.load(url, format="raw")


def test_find_zip_split_is_non_greedy(tmp_path):
    # Create a.zip containing an entry whose name includes another ".zip".
    zpath = tmp_path / "a.zip"
    with zipfile.ZipFile(zpath, "w") as zf:
        zf.writestr("b.zip/c.txt", "ok")

    ptr = data.find("a.zip/b.zip/c.txt", paths=[str(tmp_path)])
    with ptr.open() as f:
        got = f.read()
        if isinstance(got, bytes):
            got = got.decode("utf-8")
        assert got == "ok"


def test_gzip_pointer_open_enforces_sandbox(tmp_path, monkeypatch):
    """GzipFileSystemPathPointer.open() must honour the pathsec sandbox.

    Regression test for the sandbox bypass where the gzip pointer opened its
    path with GzipFile directly instead of routing through pathsec.open()
    (CWE-22 / CWE-73): it must refuse a path outside the allowed roots and an
    in-root symlink that escapes outside, while still reading in-root files.
    """
    allowed = tmp_path / "data"
    allowed.mkdir()
    outside = tmp_path / "outside"
    outside.mkdir()
    monkeypatch.setattr(pathsec, "ENFORCE", True)
    monkeypatch.setattr(pathsec, "_get_allowed_roots", lambda: {allowed.resolve()})

    # gzip file OUTSIDE the allowed root -> blocked
    secret = outside / "secret.gz"
    with gzip.open(secret, "wb") as fh:
        fh.write(b"secret")
    with pytest.raises((PermissionError, ValueError)):
        data.GzipFileSystemPathPointer(str(secret)).open().read()

    # in-root symlink escaping outside the allowed root -> blocked
    link = allowed / "link.gz"
    try:
        os.symlink(secret, link)
    except (OSError, NotImplementedError):
        pytest.skip("symlinks not supported in this environment")
    with pytest.raises((PermissionError, ValueError)):
        data.GzipFileSystemPathPointer(str(link)).open().read()

    # legitimate gzip INSIDE the allowed root -> reads (decompressed)
    good = allowed / "good.gz"
    with gzip.open(good, "wb") as fh:
        fh.write(b"hello-gz")
    with data.GzipFileSystemPathPointer(str(good)).open() as fp:
        assert fp.read() == b"hello-gz"


# ---------------------------------------------------------------------------
# retrieve() write sink must honour the pathsec sandbox (CWE-22; CVE-2026-12871)
# ---------------------------------------------------------------------------

_RETRIEVE_PAYLOAD = b"ARBITRARY-FILE-WRITE-PAYLOAD"


def test_retrieve_write_enforces_pathsec_sandbox(tmp_path, monkeypatch):
    """Under ``ENFORCE``, an out-of-root write target is rejected; in-root works.

    ``retrieve`` used the builtin ``open(..., "wb")`` for the destination, which
    bypassed pathsec and allowed arbitrary local file writes outside the sandbox.
    """
    allowed = tmp_path / "data"
    allowed.mkdir()
    outside = tmp_path / "outside"
    outside.mkdir()
    monkeypatch.setattr(pathsec, "ENFORCE", True)
    monkeypatch.setattr(pathsec, "_get_allowed_roots", lambda: {allowed.resolve()})

    # An in-root source whose bytes become the copied content.
    src = allowed / "src.txt"
    src.write_bytes(_RETRIEVE_PAYLOAD)
    url = src.resolve().as_uri()  # RFC 8089 file URL (portable across platforms)

    # Out-of-root target: the global sandbox (no required_root) raises
    # PermissionError, and nothing must be written.
    target = outside / "evil.txt"
    with pytest.raises(PermissionError):
        data.retrieve(url, filename=str(target), verbose=False)
    assert not target.exists()

    # In-root target still works and copies the bytes.
    ok = allowed / "copy.txt"
    data.retrieve(url, filename=str(ok), verbose=False)
    assert ok.read_bytes() == _RETRIEVE_PAYLOAD


def test_retrieve_write_unchanged_when_sandbox_disabled(tmp_path, monkeypatch):
    """With the sandbox off, an out-of-root target still writes (only warns)."""
    monkeypatch.setattr(pathsec, "ENFORCE", False)
    # Force *no* allowed roots so the target is unambiguously out-of-root
    # (tmp_path is usually inside tempfile.gettempdir(), which is allowed by
    # default -- that would not exercise the disabled-sandbox path).
    monkeypatch.setattr(pathsec, "_get_allowed_roots", set)
    src = tmp_path / "src.txt"
    src.write_bytes(_RETRIEVE_PAYLOAD)
    target = tmp_path / "out.txt"
    # Not enforcing: pathsec warns about the out-of-root path but still writes.
    with pytest.warns(RuntimeWarning):
        data.retrieve(src.resolve().as_uri(), filename=str(target), verbose=False)
    assert target.read_bytes() == _RETRIEVE_PAYLOAD