chore: import upstream snapshot with attribution

unstructured/file_utils/encoding.py

@@ -0,0 +1,150 @@
+from typing import IO, Optional, Tuple, Union
+
+from charset_normalizer import detect
+
+from unstructured.errors import UnprocessableEntityError
+from unstructured.partition.common.common import convert_to_bytes
+
+ENCODE_REC_THRESHOLD = 0.8
+
+# popular encodings from https://en.wikipedia.org/wiki/Popularity_of_text_encodings
+COMMON_ENCODINGS = [
+    "utf_8",
+    "iso_8859_1",
+    "iso_8859_6",
+    "iso_8859_8",
+    "ascii",
+    "big5",
+    "utf_16",
+    "utf_16_be",
+    "utf_16_le",
+    "utf_32",
+    "utf_32_be",
+    "utf_32_le",
+    "euc_jis_2004",
+    "euc_jisx0213",
+    "euc_jp",
+    "euc_kr",
+    "gb18030",
+    "shift_jis",
+    "shift_jis_2004",
+    "shift_jisx0213",
+]
+
+
+def format_encoding_str(encoding: str) -> str:
+    """Format input encoding string (e.g., `utf-8`, `iso-8859-1`, etc).
+    Parameters
+    ----------
+    encoding
+        The encoding string to be formatted (e.g., `UTF-8`, `utf_8`, `ISO-8859-1`, `iso_8859_1`,
+        etc).
+    """
+    formatted_encoding = encoding.lower().replace("_", "-")
+
+    # Special case for Arabic and Hebrew charsets with directional annotations
+    annotated_encodings = ["iso-8859-6-i", "iso-8859-6-e", "iso-8859-8-i", "iso-8859-8-e"]
+    if formatted_encoding in annotated_encodings:
+        formatted_encoding = formatted_encoding[:-2]  # remove the annotation
+
+    return formatted_encoding
+
+
+def validate_encoding(encoding: str) -> bool:
+    """Checks if an encoding string is valid. Helps to avoid errors in cases where
+    invalid encodings are extracted from malformed documents."""
+    for common_encoding in COMMON_ENCODINGS:
+        if format_encoding_str(common_encoding) == format_encoding_str(encoding):
+            return True
+    return False
+
+
+def detect_file_encoding(
+    filename: str = "",
+    file: Optional[Union[bytes, IO[bytes]]] = None,
+) -> Tuple[str, str]:
+    if filename:
+        with open(filename, "rb") as f:
+            byte_data = f.read()
+    elif file:
+        byte_data = convert_to_bytes(file)
+    else:
+        raise FileNotFoundError("No filename nor file were specified")
+
+    result = detect(byte_data)
+    encoding = result["encoding"]
+    confidence = result["confidence"]
+
+    if encoding is None or confidence is None or confidence < ENCODE_REC_THRESHOLD:
+        # Encoding detection failed, fallback to predefined encodings
+        for enc in COMMON_ENCODINGS:
+            try:
+                if filename:
+                    with open(filename, encoding=enc) as f:
+                        file_text = f.read()
+                else:
+                    file_text = byte_data.decode(enc)
+                encoding = enc
+                break
+            except (UnicodeDecodeError, UnicodeError):
+                continue
+        else:
+            # NOTE: Use UnprocessableEntityError instead of UnicodeDecodeError to avoid
+            # logging the entire file content. UnicodeDecodeError automatically stores
+            # the complete input data, which can be problematic for large files.
+            raise UnprocessableEntityError(
+                "Unable to determine file encoding after trying all common encodings. "
+                "File may be corrupted or in an unsupported format."
+            ) from None
+
+    else:
+        # NOTE: Catch UnicodeDecodeError to avoid logging the entire file content.
+        # UnicodeDecodeError automatically stores the complete input data in its
+        # 'object' attribute, which can cause issues with large files in logging
+        # and error reporting systems.
+        try:
+            file_text = byte_data.decode(encoding)
+        except (UnicodeDecodeError, UnicodeError):
+            raise UnprocessableEntityError(
+                f"File encoding detection failed: detected '{encoding}' but decode failed. "
+                f"File may be corrupted or in an unsupported format."
+            ) from None
+
+    formatted_encoding = format_encoding_str(encoding)
+
+    return formatted_encoding, file_text
+
+
+def read_txt_file(
+    filename: str = "",
+    file: Optional[Union[bytes, IO[bytes]]] = None,
+    encoding: Optional[str] = None,
+) -> Tuple[str, str]:
+    """Extracts document metadata from a plain text document."""
+    if filename:
+        if encoding:
+            formatted_encoding = format_encoding_str(encoding)
+            with open(filename, encoding=formatted_encoding) as f:
+                try:
+                    file_text = f.read()
+                except (UnicodeDecodeError, UnicodeError) as error:
+                    raise error
+        else:
+            formatted_encoding, file_text = detect_file_encoding(filename)
+    elif file:
+        if encoding:
+            formatted_encoding = format_encoding_str(encoding)
+            try:
+                file_content = file if isinstance(file, bytes) else file.read()
+                if isinstance(file_content, bytes):
+                    file_text = file_content.decode(formatted_encoding)
+                else:
+                    file_text = file_content
+            except (UnicodeDecodeError, UnicodeError) as error:
+                raise error
+        else:
+            formatted_encoding, file_text = detect_file_encoding(file=file)
+    else:
+        raise FileNotFoundError("No filename was specified")
+
+    return formatted_encoding, file_text

unstructured/file_utils/file_conversion.py

@@ -0,0 +1,81 @@
+from __future__ import annotations
+
+import os
+import re
+import tempfile
+from typing import IO
+
+from unstructured.errors import UnprocessableEntityError
+from unstructured.partition.common.common import exactly_one
+from unstructured.utils import requires_dependencies
+
+
+@requires_dependencies(["pypandoc"])
+def convert_file_to_text(filename: str, source_format: str, target_format: str) -> str:
+    """Uses pandoc to convert the source document to a raw text string."""
+    import pypandoc
+
+    try:
+        text: str = pypandoc.convert_file(
+            filename, target_format, format=source_format, sandbox=True
+        )
+    except FileNotFoundError as err:
+        msg = (
+            f"Error converting the file to text. Ensure you have the pandoc package installed on"
+            f" your system. Installation instructions are available at"
+            f" https://pandoc.org/installing.html. The original exception text was:\n{err}"
+        )
+        raise FileNotFoundError(msg)
+    except RuntimeError as err:
+        err_str = str(err)
+        if source_format == "epub" and (
+            "Couldn't extract ePub file" in err_str
+            or "No entry on path" in err_str
+            or re.search(r"exitcode ['\"]?64['\"]?", err_str)
+        ):
+            raise UnprocessableEntityError(f"Invalid EPUB file: {err_str}")
+
+        supported_source_formats, _ = pypandoc.get_pandoc_formats()
+
+        if source_format == "rtf" and source_format not in supported_source_formats:
+            additional_info = (
+                "Support for RTF files is not available in the current pandoc installation. "
+                "It was introduced in pandoc 2.14.2.\n"
+                "Reference: https://pandoc.org/releases.html#pandoc-2.14.2-2021-08-21"
+            )
+        else:
+            additional_info = ""
+
+        msg = (
+            f"{err}\n\n{additional_info}\n\n"
+            f"Current version of pandoc: {pypandoc.get_pandoc_version()}\n"
+            "Make sure you have the right version installed in your system. Please follow the"
+            " pandoc installation instructions in README.md to install the right version."
+        )
+        raise RuntimeError(msg)
+
+    return text
+
+
+def convert_file_to_html_text_using_pandoc(
+    source_format: str, filename: str | None = None, file: IO[bytes] | None = None
+) -> str:
+    """Converts a document to HTML raw text.
+
+    Enables the doucment to be processed using `partition_html()`.
+    """
+    exactly_one(filename=filename, file=file)
+
+    if file is not None:
+        with tempfile.TemporaryDirectory() as temp_dir_path:
+            tmp_file_path = os.path.join(temp_dir_path, f"tmp_file.{source_format}")
+            with open(tmp_file_path, "wb") as tmp_file:
+                tmp_file.write(file.read())
+            return convert_file_to_text(
+                filename=tmp_file_path, source_format=source_format, target_format="html"
+            )
+
+    assert filename is not None
+    return convert_file_to_text(
+        filename=filename, source_format=source_format, target_format="html"
+    )

unstructured/file_utils/filetype.py

@@ -0,0 +1,965 @@
+"""Automatically detect file-type based on inspection of the file's contents.
+
+Auto-detection proceeds via a sequence of strategies. The first strategy to confidently determine a
+file-type returns that value. A strategy that is not applicable, either because it lacks the input
+required or fails to determine a file-type, returns `None` and execution continues with the next
+strategy.
+
+`_FileTypeDetector` is the main object and implements the three strategies.
+
+The three strategies are:
+
+- Use MIME-type asserted by caller in the `content_type` argument.
+- Guess a MIME-type using libmagic, falling back to the `filetype` package when libmagic is
+  unavailable.
+- Map filename-extension to a `FileType` member.
+
+A file that fails all three strategies is assigned the value `FileType.UNK`, for "unknown".
+
+`_FileTypeDetectionContext` encapsulates the various arguments received by `detect_filetype()` and
+provides values derived from them. This object is immutable and can be passed to delegates of
+`_FileTypeDetector` to provide whatever context they need on the current detection instance.
+
+`_FileTypeDetector` delegates to _differentiator_ objects like `_ZipFileDifferentiator` for
+specialized discrimination and/or confirmation of ambiguous or frequently mis-identified
+MIME-types. Additional differentiators are planned, one for `application/x-ole-storage`
+(DOC, PPT, XLS, and MSG file-types) and perhaps others.
+"""
+
+from __future__ import annotations
+
+import contextlib
+import functools
+import importlib.util
+import io
+import json
+import os
+import re
+import tempfile
+import zipfile
+from functools import cached_property
+from typing import IO, Callable, Iterator, Optional, cast
+
+import filetype as ft
+from olefile import OleFileIO
+from oxmsg.storage import Storage
+from typing_extensions import ParamSpec
+
+from unstructured.documents.elements import Element
+from unstructured.file_utils.encoding import detect_file_encoding, format_encoding_str
+from unstructured.file_utils.model import FileType
+from unstructured.logger import logger
+from unstructured.nlp.patterns import EMAIL_HEAD_RE, LIST_OF_DICTS_PATTERN
+from unstructured.partition.common.common import add_element_metadata, exactly_one
+from unstructured.partition.common.metadata import set_element_hierarchy
+from unstructured.utils import get_call_args_applying_defaults
+
+_JSON_DISAMBIGUATION_CHUNK_SIZE = 8192
+_JSON_DISAMBIGUATION_MAX_CHARS = 1024 * 1024
+
+try:
+    importlib.import_module("magic")
+    LIBMAGIC_AVAILABLE = True
+except ImportError:
+    LIBMAGIC_AVAILABLE = False  # pyright: ignore[reportConstantRedefinition]
+
+
+def detect_filetype(
+    file_path: str | None = None,
+    file: IO[bytes] | tempfile.SpooledTemporaryFile | None = None,
+    encoding: str | None = None,
+    content_type: str | None = None,
+    metadata_file_path: Optional[str] = None,
+) -> FileType:
+    """Determine file-type of specified file using libmagic and/or fallback methods.
+
+    One of `file_path` or `file` must be specified. A `file_path` that does not
+    correspond to a file on the filesystem raises `ValueError`.
+
+    Args:
+        content_type: MIME-type of document-source, when already known. Providing
+          a value for this argument disables auto-detection unless it does not map
+          to a FileType member or is ambiguous, in which case it is ignored.
+        encoding: Only used for textual file-types. When omitted, `utf-8` is
+          assumed. Should generally be omitted except to resolve a problem with
+          textual file-types like HTML.
+        metadata_file_path: Only used when `file` is provided and then only as a
+          source for a filename-extension that may be needed as a secondary
+          content-type indicator. Ignored with the document is specified using
+          `file_path`.
+
+    Returns:
+        A member of the `FileType` enumeration, `FileType.UNK` when the file type
+        could not be determined or is not supported.
+
+    Raises:
+        ValueError: when:
+        - `file_path` is specified but does not correspond to a file on the
+          filesystem.
+        - Neither `file_path` nor `file` were specified.
+    """
+    file_buffer = file
+    if isinstance(file, tempfile.SpooledTemporaryFile):
+        file_buffer = io.BytesIO(file.read())
+        file.seek(0)
+
+    ctx = _FileTypeDetectionContext.new(
+        file_path=file_path,
+        file=file_buffer,
+        encoding=encoding,
+        content_type=content_type,
+        metadata_file_path=metadata_file_path,
+    )
+    return _FileTypeDetector.file_type(ctx)
+
+
+def is_json_processable(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    file_text: Optional[str] = None,
+    encoding: Optional[str] = "utf-8",
+) -> bool:
+    """True when file looks like a JSON array of objects.
+
+    Uses regex on a file prefix, so not entirely reliable but good enough if you already know the
+    file is JSON.
+    """
+    exactly_one(filename=filename, file=file, file_text=file_text)
+
+    if file_text is None:
+        file_text = _FileTypeDetectionContext.new(
+            file_path=filename, file=file, encoding=encoding
+        ).text_head
+
+    return re.match(LIST_OF_DICTS_PATTERN, file_text) is not None
+
+
+def is_ndjson_processable(
+    filename: Optional[str] = None,
+    file: Optional[IO[bytes]] = None,
+    file_text: Optional[str] = None,
+    encoding: Optional[str] = "utf-8",
+    allow_truncated_single_line: bool = False,
+) -> bool:
+    """True when file looks like newline-delimited JSON objects.
+
+    NDJSON is a sequence of one JSON value per line, conventionally an object on each line. A
+    payload that parses as a single JSON value (e.g. a multi-line `{...}` object or a `[...]`
+    array) is *not* NDJSON and must not be matched here, otherwise `partition_ndjson` will fail
+    later when it splits the text by lines and tries to parse each fragment.
+    """
+    exactly_one(filename=filename, file=file, file_text=file_text)
+
+    allow_truncated = allow_truncated_single_line
+    if file_text is None:
+        file_text, allow_truncated = _FileTypeDetectionContext.new(
+            file_path=filename, file=file, encoding=encoding
+        ).json_disambiguation_text
+
+    text = file_text.lstrip()
+    if not text or not text.startswith("{"):
+        return False
+
+    newline_idx = text.find("\n")
+
+    if newline_idx == -1:
+        # Single-line input. A complete `{...}` parses as a dict and is treated as 1-record
+        # NDJSON (existing tests and `partition_ndjson` rely on this). When the caller knows this
+        # is a truncated first line from a JSON-like payload, a parse failure is still compatible
+        # with a long 1-record NDJSON payload.
+        try:
+            return isinstance(json.loads(text), dict)
+        except json.JSONDecodeError:
+            return allow_truncated
+
+    # Multi-line input. NDJSON requires each record to be on its own line, so the first line
+    # must independently parse as a JSON object. A pretty-printed single JSON object has its
+    # first line be just `{` (or similar fragment) which won't parse alone — that's how we
+    # distinguish it from real NDJSON.
+    first_line = text[:newline_idx].rstrip()
+    if not first_line:
+        return False
+    try:
+        return isinstance(json.loads(first_line), dict)
+    except json.JSONDecodeError:
+        return False
+
+
+class _FileTypeDetector:
+    """Determines file type from a variety of possible inputs."""
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def file_type(cls, ctx: _FileTypeDetectionContext) -> FileType:
+        """Detect file-type of document-source described by `ctx`."""
+        return cls(ctx)._file_type
+
+    @property
+    def _file_type(self) -> FileType:
+        """FileType member corresponding to this document source."""
+        # -- An explicit content-type most commonly asserted by the client/SDK and is therefore
+        # -- inherently unreliable. On the other hand, binary file-types can be detected with 100%
+        # -- accuracy. So start with binary types and only then consider an asserted content-type,
+        # -- generally as a last resort.
+
+        if (
+            (  # strategy 1: most binary types can be detected with 100% accuracy
+                predicted_file_type := self._known_binary_file_type
+            )
+            or (  # strategy 2: use content-type asserted by caller
+                predicted_file_type := self._file_type_from_content_type
+            )
+            or (  # strategy 3: guess MIME-type using libmagic and use that
+                predicted_file_type := self._file_type_from_guessed_mime_type
+            )
+            or (  # strategy 4: use filename-extension, like ".docx" -> FileType.DOCX
+                predicted_file_type := self._file_type_from_file_extension
+            )
+        ):
+            result_file_type = predicted_file_type
+        else:
+            # give up and report FileType.UNK
+            result_file_type = FileType.UNK
+
+        if result_file_type == FileType.JSON:
+            # edge case where JSON/NDJSON content without file extension
+            # (magic lib can't distinguish them)
+            result_file_type = self._disambiguate_json_file_type
+
+        return result_file_type
+
+    @property
+    def _known_binary_file_type(self) -> FileType | None:
+        """Detect file-type for binary types we can positively detect."""
+        if file_type := _OleFileDetector.file_type(self._ctx):
+            return file_type
+
+        self._ctx.rule_out_cfb_content_types()
+
+        if file_type := _ZipFileDetector.file_type(self._ctx):
+            return file_type
+
+        self._ctx.rule_out_zip_content_types()
+
+        return None
+
+    @property
+    def _file_type_from_content_type(self) -> FileType | None:
+        """Map passed content-type argument to a file-type, subject to certain rules."""
+
+        # -- when no content-type was asserted by caller, this strategy is not applicable --
+        if not self._ctx.content_type:
+            return None
+
+        # -- otherwise we trust the passed `content_type` as long as `FileType` recognizes it --
+        return FileType.from_mime_type(self._ctx.content_type)
+
+    @property
+    def _disambiguate_json_file_type(self) -> FileType:
+        """Disambiguate JSON/NDJSON file-type based on file contents.
+
+        NDJSON is detected first because it has the strictest signature (multiple JSON values
+        separated by newlines, with the first line independently parsable). Anything else that
+        libmagic flagged as JSON is classified as `FileType.JSON`; the JSON partitioner has its
+        own `is_json_processable` schema check and will reject non-conforming payloads with a
+        clear error.
+        """
+        file_text, allow_truncated_single_line = self._ctx.json_disambiguation_text
+        if is_ndjson_processable(
+            file_text=file_text,
+            allow_truncated_single_line=allow_truncated_single_line,
+        ):
+            return FileType.NDJSON
+        return FileType.JSON
+
+    @property
+    def _file_type_from_guessed_mime_type(self) -> FileType | None:
+        """FileType based on auto-detection of MIME-type by libmagic.
+
+        In some cases refinements are necessary on the magic-derived MIME-types. This process
+        includes applying those rules, most of which are accumulated through practical experience.
+        """
+        mime_type = self._ctx.mime_type
+        extension = self._ctx.extension
+
+        # -- when libmagic is not installed, the `filetype` package is used instead.
+        # -- `filetype.guess()` returns `None` for file-types it does not support, which
+        # -- unfortunately includes all the textual file-types like CSV, EML, HTML, MD, RST, RTF,
+        # -- TSV, and TXT. When we have no guessed MIME-type, this strategy is not applicable.
+        if mime_type is None:
+            return None
+
+        if mime_type.endswith("xml"):
+            return FileType.HTML if extension in (".html", ".htm") else FileType.XML
+
+        if differentiator := _TextFileDifferentiator.applies(self._ctx):
+            return differentiator.file_type
+
+        # -- All source-code files (e.g. *.py, *.js) are classified as plain text for the moment --
+        if self._ctx.has_code_mime_type:
+            return FileType.TXT
+
+        if mime_type.endswith("empty"):
+            return FileType.EMPTY
+
+        if mime_type.endswith("json") and self._ctx.extension == ".ndjson":
+            return FileType.NDJSON
+
+        # -- if no more-specific rules apply, use the MIME-type -> FileType mapping when present --
+        file_type = FileType.from_mime_type(mime_type)
+        if file_type != FileType.UNK:
+            return file_type
+
+        # -- on some environments libmagic can return a generic/unhelpful MIME-type
+        # -- like octet-stream") for files that the `filetype` package identify.
+        # -- when that happens we retry using `filetype`  `FileType.UNK` results.
+        if LIBMAGIC_AVAILABLE:
+            fallback_mime_type = (
+                ft.guess_mime(self._ctx.file_path)
+                if self._ctx.file_path
+                else ft.guess_mime(self._ctx.file_head)
+            )
+            fallback_file_type = FileType.from_mime_type(fallback_mime_type)
+            if fallback_file_type and fallback_file_type != FileType.UNK:
+                return fallback_file_type
+
+        return None
+
+    @cached_property
+    def _file_type_from_file_extension(self) -> FileType | None:
+        """Determine file-type from filename extension.
+
+        Returns `None` when no filename is available or when the extension does not map to a
+        supported file-type.
+        """
+        return FileType.from_extension(self._ctx.extension)
+
+
+class _FileTypeDetectionContext:
+    """Provides all arguments to auto-file detection and values derived from them.
+
+    NOTE that `._content_type` is mutable via `.rule_out_*_content_types()` methods, so it should
+    not be assumed to be a constant value across those calls.
+
+    This keeps computation of derived values out of the file-detection code but more importantly
+    allows the main filetype-detector to pass the full context to any delegates without coupling
+    itself to which values it might need.
+    """
+
+    def __init__(
+        self,
+        file_path: str | None = None,
+        *,
+        file: IO[bytes] | None = None,
+        encoding: str | None = None,
+        content_type: str | None = None,
+        metadata_file_path: str | None = None,
+    ):
+        self._file_path_arg = file_path
+        self._file_arg = file
+        self._encoding_arg = encoding
+        self._content_type = content_type
+        self._metadata_file_path = metadata_file_path
+
+    @classmethod
+    def new(
+        cls,
+        *,
+        file_path: str | None,
+        file: IO[bytes] | None,
+        encoding: str | None,
+        content_type: str | None = None,
+        metadata_file_path: str | None = None,
+    ) -> _FileTypeDetectionContext:
+        self = cls(
+            file_path=file_path,
+            file=file,
+            encoding=encoding,
+            content_type=content_type,
+            metadata_file_path=metadata_file_path,
+        )
+        self._validate()
+        return self
+
+    @property
+    def content_type(self) -> str | None:
+        """MIME-type asserted by caller; not based on inspection of file by this process.
+
+        Would commonly occur when the file was downloaded via HTTP and a `"Content-Type:` header was
+        present on the response. These are often ambiguous and sometimes just wrong so get some
+        further verification. All lower-case when not `None`.
+        """
+        # -- Note `._content_type` is mutable via `.invalidate_content_type()` so this cannot be a
+        # -- `@cached_property`.
+        return self._content_type.lower() if self._content_type else None
+
+    @cached_property
+    def encoding(self) -> str:
+        """Character-set used to encode text of this file.
+
+        Relevant for textual file-types only, like HTML, TXT, JSON, etc.
+        """
+        return format_encoding_str(self._encoding_arg or "utf-8")
+
+    @cached_property
+    def extension(self) -> str:
+        """Best filename-extension we can muster, "" when there is no available source."""
+        # -- get from file_path, or file when it has a name (path) --
+        with self.open() as file:
+            if hasattr(file, "name") and file.name:
+                return os.path.splitext(file.name)[1].lower()
+
+        # -- otherwise use metadata file-path when provided --
+        if file_path := self._metadata_file_path:
+            return os.path.splitext(file_path)[1].lower()
+
+        # -- otherwise empty str means no extension, same as a path like "a/b/name-no-ext" --
+        return ""
+
+    @cached_property
+    def file_head(self) -> bytes:
+        """The initial bytes of the file to be recognized, for use with libmagic detection."""
+        with self.open() as file:
+            return file.read(8192)
+
+    @cached_property
+    def file_path(self) -> str | None:
+        """Filesystem path to file to be inspected, when provided on call.
+
+        None when the caller specified the source as a file-like object instead. Useful for user
+        feedback on an error, but users of context should have little use for it otherwise.
+        """
+        if (file_path := self._file_path_arg) is None:
+            return None
+
+        return os.path.realpath(file_path) if os.path.islink(file_path) else file_path
+
+    @cached_property
+    def has_code_mime_type(self) -> bool:
+        """True when `mime_type` plausibly indicates a programming language source-code file."""
+        mime_type = self.mime_type
+
+        if mime_type is None:
+            return False
+
+        # -- check Go separately to avoid matching other MIME type containing "go" --
+        if mime_type == "text/x-go":
+            return True
+
+        return any(
+            lang in mime_type
+            for lang in [
+                "c#",
+                "c++",
+                "cpp",
+                "csharp",
+                "java",
+                "javascript",
+                "php",
+                "python",
+                "ruby",
+                "swift",
+                "typescript",
+            ]
+        )
+
+    @cached_property
+    def is_zipfile(self) -> bool:
+        """True when file is a Zip archive."""
+        with self.open() as file:
+            return zipfile.is_zipfile(file)
+
+    @cached_property
+    def mime_type(self) -> str | None:
+        """The best MIME-type we can get from `magic` (or `filetype` package).
+
+        A `str` return value is always in lower-case.
+        """
+        file_path = self.file_path
+
+        if LIBMAGIC_AVAILABLE:
+            import magic
+
+            magic_mime = (
+                magic.from_file(file_path, mime=True)
+                if file_path
+                else magic.from_buffer(self.file_head, mime=True)
+            )
+            magic_mime = magic_mime.lower() if magic_mime else None
+
+            # When libmagic returns None or "application/octet-stream", try the filetype package
+            # (magic-byte signatures) for formats libmagic often mis-detects (e.g. BMP, HEIC, WAV).
+            if magic_mime and magic_mime != "application/octet-stream":
+                return magic_mime
+
+            ft_mime = ft.guess_mime(file_path) if file_path else ft.guess_mime(self.file_head)
+            if ft_mime:
+                return ft_mime.lower()
+            # filetype could not identify; same outcome as if we had not tried it.
+            return magic_mime
+
+        mime_type = ft.guess_mime(file_path) if file_path else ft.guess_mime(self.file_head)
+
+        if mime_type is None:
+            logger.warning(
+                "libmagic is unavailable but assists in filetype detection. Please consider"
+                " installing libmagic for better results."
+            )
+            return None
+
+        return mime_type.lower()
+
+    @contextlib.contextmanager
+    def open(self) -> Iterator[IO[bytes]]:
+        """Encapsulates complexity of dealing with file-path or file-like-object.
+
+        Provides an `IO[bytes]` object as the "common-denominator" document source.
+
+        Must be used as a context manager using a `with` statement:
+
+            with self._file as file:
+                do things with file
+
+        File is guaranteed to be at read position 0 when called.
+        """
+        if self.file_path:
+            with open(self.file_path, "rb") as f:
+                yield f
+        else:
+            file = self._file_arg
+            assert file is not None  # -- guaranteed by `._validate()` --
+            file.seek(0)
+            yield file
+
+    def rule_out_cfb_content_types(self) -> None:
+        """Invalidate content-type when a legacy MS-Office file-type is asserted.
+
+        Used before returning `None`; at that point we know the file is not one of these formats
+        so if the asserted `content-type` is a legacy MS-Office type we know it's wrong and should
+        not be used as a fallback later in the detection process.
+        """
+        if FileType.from_mime_type(self._content_type) in (
+            FileType.DOC,
+            FileType.MSG,
+            FileType.PPT,
+            FileType.XLS,
+        ):
+            self._content_type = None
+
+    def rule_out_zip_content_types(self) -> None:
+        """Invalidate content-type when an MS-Office 2007+ file-type is asserted.
+
+        Used before returning `None`; at that point we know the file is not one of these formats
+        so if the asserted `content-type` is an MS-Office 2007+ type we know it's wrong and should
+        not be used as a fallback later in the detection process.
+        """
+        if FileType.from_mime_type(self._content_type) in (
+            FileType.DOCX,
+            FileType.EPUB,
+            FileType.ODT,
+            FileType.PPTX,
+            FileType.XLSX,
+            FileType.ZIP,
+        ):
+            self._content_type = None
+
+    @cached_property
+    def text_head(self) -> str:
+        """The initial characters of the text file for use with text-format differentiation.
+
+        Raises:
+            UnicodeDecodeError if file cannot be read as text.
+        """
+        # TODO: only attempts fallback character-set detection for file-path case, not for
+        # file-like object case. Seems like we should do both.
+
+        if file := self._file_arg:
+            file.seek(0)
+            content = file.read(4096)
+            file.seek(0)
+            return (
+                content
+                if isinstance(content, str)
+                else content.decode(encoding=self.encoding, errors="ignore")
+            )
+
+        file_path = self.file_path
+        assert file_path is not None  # -- guaranteed by `._validate` --
+
+        try:
+            with open(file_path, encoding=self.encoding) as f:
+                return f.read(4096)
+        except UnicodeDecodeError:
+            encoding, _ = detect_file_encoding(filename=file_path)
+            with open(file_path, encoding=encoding) as f:
+                return f.read(4096)
+
+    @cached_property
+    def json_disambiguation_text(self) -> tuple[str, bool]:
+        """Text prefix for JSON/NDJSON disambiguation and whether the first line was truncated."""
+
+        if file := self._file_arg:
+            file.seek(0)
+            content, first_line_truncated = self._read_until_newline_or_limit(file)
+            file.seek(0)
+            if isinstance(content, str):
+                return content, first_line_truncated
+            return content.decode(encoding=self.encoding, errors="ignore"), first_line_truncated
+
+        file_path = self.file_path
+        assert file_path is not None  # -- guaranteed by `._validate` --
+
+        try:
+            with open(file_path, encoding=self.encoding) as f:
+                content, first_line_truncated = self._read_until_newline_or_limit(f)
+                assert isinstance(content, str)
+                return content, first_line_truncated
+        except UnicodeDecodeError:
+            encoding, _ = detect_file_encoding(filename=file_path)
+            with open(file_path, encoding=encoding) as f:
+                content, first_line_truncated = self._read_until_newline_or_limit(f)
+                assert isinstance(content, str)
+                return content, first_line_truncated
+
+    def _validate(self) -> None:
+        """Raise if the context is invalid."""
+        if self.file_path and not os.path.isfile(self.file_path):
+            raise FileNotFoundError(f"no such file {self._file_path_arg}")
+        if not self.file_path and not self._file_arg:
+            raise ValueError("either `file_path` or `file` argument must be provided")
+
+    @staticmethod
+    def _read_until_newline_or_limit(file: IO) -> tuple[str | bytes, bool]:
+        """Read through the first newline, stopping at a bounded prefix if none is found."""
+        chunks: list[str | bytes] = []
+        chars_read = 0
+
+        while chars_read < _JSON_DISAMBIGUATION_MAX_CHARS:
+            chars_to_read = min(
+                _JSON_DISAMBIGUATION_CHUNK_SIZE,
+                _JSON_DISAMBIGUATION_MAX_CHARS - chars_read,
+            )
+            chunk = file.read(chars_to_read)
+            if not chunk:
+                return _FileTypeDetectionContext._join_text_chunks(chunks), False
+
+            newline = b"\n" if isinstance(chunk, bytes) else "\n"
+            newline_idx = chunk.find(newline)
+            if newline_idx != -1:
+                chunks.append(chunk[: newline_idx + 1])
+                return _FileTypeDetectionContext._join_text_chunks(chunks), False
+
+            chunks.append(chunk)
+            chars_read += len(chunk)
+
+        return _FileTypeDetectionContext._join_text_chunks(chunks), True
+
+    @staticmethod
+    def _join_text_chunks(chunks: list[str | bytes]) -> str | bytes:
+        """Join chunks without mixing text and bytes types."""
+        if chunks and isinstance(chunks[0], bytes):
+            return b"".join(cast(list[bytes], chunks))
+        return "".join(cast(list[str], chunks))
+
+
+class _OleFileDetector:
+    """Detect and differentiate a CFB file, aka. "OLE" file.
+
+    Compound File Binary Format (CFB), aka. OLE file, is use by Microsoft for legacy MS Office
+    files (DOC, PPT, XLS) as well as for Outlook MSG files.
+    """
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def file_type(cls, ctx: _FileTypeDetectionContext) -> FileType | None:
+        """Specific file-type when file is a CFB file, `None` otherwise."""
+        return cls(ctx)._file_type
+
+    @property
+    def _file_type(self) -> FileType | None:
+        """Differentiated file-type for Microsoft Compound File Binary Format (CFBF).
+
+        Returns one of:
+        - `FileType.DOC`
+        - `FileType.PPT`
+        - `FileType.XLS`
+        - `FileType.MSG`
+        - `None` when the file is not one of these.
+        """
+        # -- all CFB files share common magic number, start with that --
+        if not self._is_ole_file:
+            return None
+
+        # -- check storage contents of the ole file for file-type specific stream names --
+        if (ole_file_type := self._ole_file_type) is not None:
+            return ole_file_type
+
+        return None
+
+    @cached_property
+    def _is_ole_file(self) -> bool:
+        """True when file has CFB magic first 8 bytes."""
+        with self._ctx.open() as file:
+            return file.read(8) == b"\xd0\xcf\x11\xe0\xa1\xb1\x1a\xe1"
+
+    @cached_property
+    def _ole_file_type(self) -> FileType | None:
+        with self._ctx.open() as f:
+            ole = OleFileIO(f)  # pyright: ignore[reportUnknownVariableType]
+            root_storage = Storage.from_ole(ole)  # pyright: ignore[reportUnknownMemberType]
+
+        for stream in root_storage.streams:
+            if stream.name == "WordDocument":
+                return FileType.DOC
+            elif stream.name == "PowerPoint Document":
+                return FileType.PPT
+            elif stream.name == "Workbook":
+                return FileType.XLS
+            elif stream.name == "__properties_version1.0":
+                return FileType.MSG
+
+        return None
+
+
+class _TextFileDifferentiator:
+    """Refine a textual file-type that may not be as specific as it could be."""
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def applies(cls, ctx: _FileTypeDetectionContext) -> _TextFileDifferentiator | None:
+        """Constructs an instance, but only if this differentiator applies in `ctx`."""
+        mime_type = ctx.mime_type
+        return (
+            cls(ctx)
+            if mime_type and (mime_type == "message/rfc822" or mime_type.startswith("text"))
+            else None
+        )
+
+    @cached_property
+    def file_type(self) -> FileType:
+        """Differentiated file-type for textual content.
+
+        Always produces a file-type, worst case that's `FileType.TXT` when nothing more specific
+        applies.
+        """
+        extension = self._ctx.extension
+
+        if extension in [
+            ".csv",
+            ".eml",
+            ".html",
+            ".json",
+            ".markdown",
+            ".md",
+            ".org",
+            ".p7s",
+            ".rst",
+            ".rtf",
+            ".tab",
+            ".tsv",
+        ]:
+            return FileType.from_extension(extension) or FileType.TXT
+
+        # NOTE(crag): for older versions of the OS libmagic package, such as is currently
+        # installed on the Unstructured docker image, .json files resolve to "text/plain"
+        # rather than "application/json". this corrects for that case.
+        if self._is_json:
+            return FileType.JSON
+
+        if self._is_csv:
+            return FileType.CSV
+
+        if self._is_eml:
+            return FileType.EML
+
+        if extension in (".text", ".txt"):
+            return FileType.TXT
+
+        # Safety catch
+        if file_type := FileType.from_mime_type(self._ctx.mime_type):
+            return file_type
+
+        return FileType.TXT
+
+    @cached_property
+    def _is_csv(self) -> bool:
+        """True when file is plausibly in Comma Separated Values (CSV) format."""
+
+        def count_commas(text: str):
+            """Counts the number of commas in a line, excluding commas in quotes."""
+            pattern = r"(?=(?:[^\"]*\"[^\"]*\")*[^\"]*$),"
+            matches = re.findall(pattern, text)
+            return len(matches)
+
+        lines = self._ctx.text_head.strip().splitlines()
+        if len(lines) < 2:
+            return False
+        # -- check at most the first 10 lines --
+        lines = lines[: len(lines)] if len(lines) < 10 else lines[:10]
+        # -- any lines without at least one comma disqualifies the file --
+        if any("," not in line for line in lines):
+            return False
+        header_count = count_commas(lines[0])
+        return all(count_commas(line) == header_count for line in lines[1:])
+
+    @cached_property
+    def _is_eml(self) -> bool:
+        """Checks if a text/plain file is actually a .eml file.
+
+        Uses a regex pattern to see if the start of the file matches the typical pattern for a .eml
+        file.
+        """
+        return EMAIL_HEAD_RE.match(self._ctx.text_head) is not None
+
+    @cached_property
+    def _is_json(self) -> bool:
+        """True when file is JSON collection.
+
+        A JSON file that contains only a string, number, or boolean, while valid JSON, will fail
+        this test since it is not partitionable.
+        """
+        text_head = self._ctx.text_head
+
+        # -- an empty file is not JSON --
+        if not text_head.lstrip():
+            return False
+
+        # -- has to be a list or object, no string, number, or bool --
+        if text_head.lstrip()[0] not in "[{":
+            return False
+
+        try:
+            with self._ctx.open() as file:
+                json.load(file)
+            return True
+        except json.JSONDecodeError:
+            return False
+
+
+class _ZipFileDetector:
+    """Detect and differentiate a Zip-archive file."""
+
+    def __init__(self, ctx: _FileTypeDetectionContext):
+        self._ctx = ctx
+
+    @classmethod
+    def file_type(cls, ctx: _FileTypeDetectionContext) -> FileType | None:
+        """Most specific file-type available when file is a Zip file, `None` otherwise.
+
+        MS-Office 2007+ files are detected with 100% accuracy. Otherwise this returns `None`, even
+        when we can tell it's a Zip file, so later strategies can have a crack at it. In
+        particular, ODT and EPUB files are Zip archives but are not detected here.
+        """
+        return cls(ctx)._file_type
+
+    @cached_property
+    def _file_type(self) -> FileType | None:
+        """Differentiated file-type for a Zip archive.
+
+        Returns `FileType.DOCX`, `FileType.PPTX`, or `FileType.XLSX` when one of those applies,
+        `None` otherwise.
+        """
+        if not self._ctx.is_zipfile:
+            return None
+
+        with self._ctx.open() as file:
+            zip = zipfile.ZipFile(file)
+
+            filenames = zip.namelist()
+
+            if any(re.match(r"word/document.*\.xml$", filename) for filename in filenames):
+                return FileType.DOCX
+
+            if any(re.match(r"xl/workbook.*\.xml$", filename) for filename in filenames):
+                return FileType.XLSX
+
+            if any(re.match(r"ppt/presentation.*\.xml$", filename) for filename in filenames):
+                return FileType.PPTX
+
+            # -- ODT and EPUB files place their MIME-type in `mimetype` in the archive root --
+            if "mimetype" in filenames:
+                with zip.open("mimetype") as f:
+                    mime_type = f.read().decode("utf-8").strip()
+                    return FileType.from_mime_type(mime_type)
+
+        return FileType.ZIP
+
+
+_P = ParamSpec("_P")
+
+
+def add_metadata(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+    @functools.wraps(func)
+    def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
+        elements = func(*args, **kwargs)
+        call_args = get_call_args_applying_defaults(func, *args, **kwargs)
+
+        if call_args.get("metadata_filename"):
+            call_args["filename"] = call_args.get("metadata_filename")
+
+        metadata_kwargs = {
+            kwarg: call_args.get(kwarg) for kwarg in ("filename", "url", "text_as_html")
+        }
+        # NOTE (yao): do not use cast here as cast(None) still is None
+        if not str(kwargs.get("model_name", "")).startswith("chipper"):
+            # NOTE(alan): Skip hierarchy if using chipper, as it should take care of that
+            elements = set_element_hierarchy(elements)
+
+        for element in elements:
+            # NOTE(robinson) - Attached files have already run through this logic
+            # in their own partitioning function
+            if element.metadata.attached_to_filename is None:
+                add_element_metadata(element, **metadata_kwargs)
+
+        return elements
+
+    return wrapper
+
+
+def add_filetype(
+    filetype: FileType,
+) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
+    """Post-process element-metadata for list[Element] from partitioning.
+
+    This decorator adds a post-processing step to a document partitioner.
+
+    - Adds `.metadata.filetype` (source-document MIME-type) metadata value
+
+    This "partial" decorator is present because `partition_image()` does not apply
+    `.metadata.filetype` this way since each image type has its own MIME-type (e.g. `image.jpeg`,
+    `image/png`, etc.).
+    """
+
+    def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+        @functools.wraps(func)
+        def wrapper(*args: _P.args, **kwargs: _P.kwargs) -> list[Element]:
+            elements = func(*args, **kwargs)
+
+            for element in elements:
+                # NOTE(robinson) - Attached files have already run through this logic
+                # in their own partitioning function
+                if element.metadata.attached_to_filename is None:
+                    add_element_metadata(element, filetype=filetype.mime_type)
+
+            return elements
+
+        return wrapper
+
+    return decorator
+
+
+def add_metadata_with_filetype(
+    filetype: FileType,
+) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
+    """..."""
+
+    def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+        return add_filetype(filetype=filetype)(add_metadata(func))
+
+    return decorator

unstructured/file_utils/google_filetype.py

@@ -0,0 +1,9 @@
+GOOGLE_DRIVE_EXPORT_TYPES = {
+    "application/vnd.google-apps.document": "application/"
+    "vnd.openxmlformats-officedocument.wordprocessingml.document",
+    "application/vnd.google-apps.spreadsheet": "application/"
+    "vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+    "application/vnd.google-apps.presentation": "application/"
+    "vnd.openxmlformats-officedocument.presentationml.presentation",
+    "application/vnd.google-apps.photo": "image/jpeg",
+}

unstructured/file_utils/model.py

@@ -0,0 +1,591 @@
+"""Domain-model for file-types."""
+
+from __future__ import annotations
+
+import enum
+from typing import TYPE_CHECKING, Callable, Iterable, Type, cast
+
+from typing_extensions import ParamSpec
+
+if TYPE_CHECKING:
+    from unstructured.documents.elements import Element
+else:
+    Element = None
+
+
+def _create_file_type_enum(
+    cls: Type["FileType"],
+    value: str,
+    partitioner_shortname: str | None,
+    importable_package_dependencies: Iterable[str],
+    extra_name: str | None,
+    extensions: Iterable[str],
+    canonical_mime_type: str,
+    alias_mime_types: Iterable[str],
+    partitioner_full_module_path: str | None = None,
+) -> "FileType":
+    """
+    Moving here instead of directly in the FileType.__new__ allows us
+    to dynamically create new enum properties.
+
+    FileType.__new__ does not work with dynamic properties.
+    """
+    val = object.__new__(cls)
+    val._value_ = value
+    val._partitioner_shortname = partitioner_shortname
+    val._importable_package_dependencies = tuple(importable_package_dependencies)
+    val._extra_name = extra_name
+    val._extensions = tuple(extensions)
+    val._canonical_mime_type = canonical_mime_type
+    val._alias_mime_types = tuple(alias_mime_types)
+    val._partitioner_full_module_path = partitioner_full_module_path
+    return val
+
+
+class FileType(enum.Enum):
+    """The collection of file-types recognized by `unstructured`.
+
+    Note not all of these can be partitioned, e.g. ZIP has no partitioner.
+    """
+
+    _partitioner_shortname: str | None
+    """Like "docx", from which partitioner module and function-name can be derived via template."""
+
+    _importable_package_dependencies: tuple[str, ...]
+    """Packages that must be available for import for this file-type's partitioner to work."""
+
+    _extra_name: str | None
+    """`pip install` extra that provides package dependencies for this file-type."""
+
+    _extensions: tuple[str, ...]
+    """Filename-extensions recognized as this file-type. Use for secondary identification only."""
+
+    _canonical_mime_type: str
+    """The MIME-type used as `.metadata.filetype` for this file-type."""
+
+    _alias_mime_types: tuple[str, ...]
+    """MIME-types accepted as identifying this file-type."""
+
+    _partitioner_full_module_path: str | None
+    """Fully-qualified name of module providing partitioner for this file-type."""
+
+    def __new__(
+        cls,
+        value: str,
+        partitioner_shortname: str | None,
+        importable_package_dependencies: Iterable[str],
+        extra_name: str | None,
+        extensions: Iterable[str],
+        canonical_mime_type: str,
+        alias_mime_types: Iterable[str],
+        partitioner_full_module_path: str | None = None,
+    ):
+        return _create_file_type_enum(
+            cls,
+            value,
+            partitioner_shortname,
+            importable_package_dependencies,
+            extra_name,
+            extensions,
+            canonical_mime_type,
+            alias_mime_types,
+            partitioner_full_module_path,
+        )
+
+    def __lt__(self, other: FileType) -> bool:
+        """Makes `FileType` members comparable with relational operators, at least with `<`.
+
+        This makes them sortable, in particular it supports sorting for pandas groupby functions.
+        """
+        return self.name < other.name
+
+    @classmethod
+    def from_extension(cls, extension: str | None) -> FileType | None:
+        """Select a FileType member based on an extension.
+
+        `extension` must include the leading period, like `".pdf"`. Extension is suitable as a
+        secondary file-type identification method but is unreliable for primary identification.
+
+        Returns `None` when `extension` is not registered for any supported file-type.
+        """
+        if extension in (None, "", "."):
+            return None
+        # -- not super efficient but plenty fast enough for once-or-twice-per-file use and avoids
+        # -- limitations on defining a class variable on an Enum.
+        for m in cls.__members__.values():
+            if extension in m._extensions:
+                return m
+        return None
+
+    @classmethod
+    def from_mime_type(cls, mime_type: str | None) -> FileType | None:
+        """Select a FileType member based on a MIME-type.
+
+        Returns `None` when `mime_type` is `None` or does not map to the canonical MIME-type of a
+        `FileType` member or one of its alias MIME-types.
+        """
+        if mime_type is None:
+            return None
+        # -- not super efficient but plenty fast enough for once-or-twice-per-file use and avoids
+        # -- limitations on defining a class variable on an Enum.
+        for m in cls.__members__.values():
+            if mime_type == m._canonical_mime_type or mime_type in m._alias_mime_types:
+                return m
+        return None
+
+    @property
+    def extra_name(self) -> str | None:
+        """The `pip` "extra" that must be installed to provide this file-type's dependencies.
+
+        Like "image" for PNG, as in `pip install "unstructured[image]"`.
+
+        `None` when partitioning this file-type requires only the base `unstructured` install.
+        """
+        return self._extra_name
+
+    @property
+    def importable_package_dependencies(self) -> tuple[str, ...]:
+        """Packages that must be importable for this file-type's partitioner to work.
+
+        In general, these are the packages provided by the `pip install` "extra" for this file-type,
+        like `pip install "unstructured[docx]"` loads the `python-docx` package.
+
+        Note that these names are the ones used in an `import` statement, which is not necessarily
+        the same as the _distribution_ package name used by `pip`. For example, the DOCX
+        distribution package name is `"python-docx"` whereas the _importable_ package name is
+        `"docx"`. This latter name as it appears like `import docx` is what is provided by this
+        property.
+
+        The return value is an empty tuple for file-types that do not require optional dependencies.
+
+        Note this property does not complain when accessed on a non-partitionable file-type, it
+        simply returns an empty tuple because file-types that are not partitionable require no
+        optional dependencies.
+        """
+        return self._importable_package_dependencies
+
+    @property
+    def is_partitionable(self) -> bool:
+        """True when there is a partitioner for this file-type.
+
+        Note this does not check whether the dependencies for this file-type are installed so
+        attempting to partition a file of this type may still fail. This is meant for
+        distinguishing file-types like ZIP, EMPTY, and UNK which are legitimate file-types
+        but have no associated partitioner.
+        """
+        return bool(self._partitioner_shortname) or bool(self._partitioner_full_module_path)
+
+    @property
+    def mime_type(self) -> str:
+        """The canonical MIME-type for this file-type, suitable for use in metadata.
+
+        This value is used in `.metadata.filetype` for elements partitioned from files of this
+        type. In general it is the "offical", "recommended", or "defacto-standard" MIME-type for
+        files of this type, in that order, as available.
+        """
+        return self._canonical_mime_type
+
+    @property
+    def partitioner_function_name(self) -> str:
+        """Name of partitioner function for this file-type. Like "partition_docx".
+
+        Raises when this property is accessed on a file-type that is not partitionable. Use
+        `.is_partitionable` to avoid exceptions when partitionability is unknown.
+        """
+        # -- Raise when this property is accessed on a FileType member that has no partitioner
+        # -- shortname. This prevents a harder-to-find bug from appearing far away from this call
+        # -- when code would try to `getattr(module, None)` or whatever.
+        if full_module_path := self._partitioner_full_module_path:
+            return full_module_path.split(".")[-1]
+
+        if (shortname := self._partitioner_shortname) is None:
+            raise ValueError(
+                f"`.partitioner_function_name` is undefined because FileType.{self.name} is not"
+                f" partitionable. Use `.is_partitionable` to determine whether a `FileType`"
+                f" is partitionable."
+            )
+        return f"partition_{shortname}"
+
+    @property
+    def partitioner_module_qname(self) -> str:
+        """Fully-qualified name of module providing partitioner for this file-type.
+
+        e.g. "unstructured.partition.docx" for FileType.DOCX.
+        """
+        # -- Raise when this property is accessed on a FileType member that has no partitioner
+        # -- shortname. This prevents a harder-to-find bug from appearing far away from this call
+        # -- when code would try to `importlib.import_module(None)` or whatever.
+        if full_module_path := self._partitioner_full_module_path:
+            return ".".join(full_module_path.split(".")[:-1])
+
+        if (shortname := self._partitioner_shortname) is None:
+            raise ValueError(
+                f"`.partitioner_module_qname` is undefined because FileType.{self.name} is not"
+                f" partitionable. Use `.is_partitionable` to determine whether a `FileType`"
+                f" is partitionable."
+            )
+        return f"unstructured.partition.{shortname}"
+
+    @property
+    def partitioner_shortname(self) -> str | None:
+        """Familiar name of partitioner, like "image" for file-types that use `partition_image()`.
+
+        One use is to determine whether a file-type is one of the five image types, all of which
+        are processed by `partition_image()`.
+
+        `None` for file-types that are not partitionable, although `.is_partitionable` is the
+        preferred way of discovering that.
+        """
+        return self._partitioner_shortname
+
+    BMP = (
+        "bmp",  # -- value for this Enum member, like BMP = "bmp" in a simple enum --
+        "image",  # -- partitioner_shortname --
+        ["unstructured_inference"],  # -- importable_package_dependencies --
+        "image",  # -- extra_name - like `pip install "unstructured[image]"` in this case --
+        [".bmp"],  # -- extensions - filename extensions that map to this file-type --
+        "image/bmp",  # -- canonical_mime_type -  MIME-type written to `.metadata.filetype` --
+        [
+            "image/x-bmp",
+            "image/x-ms-bmp",  # returned by older libmagic versions
+        ],
+    )
+    CSV = (
+        "csv",
+        "csv",
+        ["pandas"],
+        "csv",
+        [".csv"],
+        "text/csv",
+        [
+            "application/csv",
+            "application/x-csv",
+            "text/comma-separated-values",
+            "text/x-comma-separated-values",
+            "text/x-csv",
+        ],
+    )
+    DOC = ("doc", "doc", ["docx"], "doc", [".doc"], "application/msword", cast(list[str], []))
+    DOCX = (
+        "docx",
+        "docx",
+        ["docx"],
+        "docx",
+        [".docx"],
+        "application/vnd.openxmlformats-officedocument.wordprocessingml.document",
+        cast(list[str], []),
+    )
+    EML = (
+        "eml",
+        "email",
+        cast(list[str], []),
+        None,
+        [".eml", ".p7s"],
+        "message/rfc822",
+        cast(list[str], []),
+    )
+    EPUB = (
+        "epub",
+        "epub",
+        ["pypandoc"],
+        "epub",
+        [".epub"],
+        "application/epub",
+        ["application/epub+zip"],
+    )
+    FLAC = (
+        "flac",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".flac"],
+        "audio/flac",
+        ["audio/x-flac"],
+    )
+    HEIC = (
+        "heic",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".heic"],
+        "image/heic",
+        ["image/x-heic"],
+    )
+    HTML = (
+        "html",
+        "html",
+        cast(list[str], []),
+        None,
+        [".html", ".htm"],
+        "text/html",
+        cast(list[str], []),
+    )
+    JPG = (
+        "jpg",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".jpeg", ".jpg"],
+        "image/jpeg",
+        cast(list[str], []),
+    )
+    JSON = (
+        "json",
+        "json",
+        cast(list[str], []),
+        None,
+        [".json"],
+        "application/json",
+        cast(list[str], []),
+    )
+    M4A = (
+        "m4a",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".m4a"],
+        "audio/mp4",
+        ["audio/x-m4a"],
+    )
+    MD = ("md", "md", ["markdown"], "md", [".md"], "text/markdown", ["text/x-markdown"])
+    MP3 = (
+        "mp3",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".mp3"],
+        "audio/mpeg",
+        ["audio/mp3", "audio/x-mp3", "audio/x-mpeg"],
+    )
+    MSG = (
+        "msg",
+        "msg",
+        ["oxmsg"],
+        "msg",
+        [".msg"],
+        "application/vnd.ms-outlook",
+        cast(list[str], []),
+    )
+    NDJSON = (
+        "ndjson",
+        "ndjson",
+        cast(list[str], []),
+        None,
+        [".ndjson"],
+        "application/x-ndjson",
+        cast(list[str], []),
+    )
+    ODT = (
+        "odt",
+        "odt",
+        ["docx", "pypandoc"],
+        "odt",
+        [".odt"],
+        "application/vnd.oasis.opendocument.text",
+        cast(list[str], []),
+    )
+    OGG = (
+        "ogg",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".ogg", ".oga"],
+        "audio/ogg",
+        ["audio/x-ogg"],
+    )
+    OPUS = (
+        "opus",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".opus"],
+        "audio/opus",
+        cast(list[str], []),
+    )
+    ORG = ("org", "org", ["pypandoc"], "org", [".org"], "text/org", cast(list[str], []))
+    PDF = (
+        "pdf",
+        "pdf",
+        ["pdf2image", "pdfminer", "PIL"],
+        "pdf",
+        [".pdf"],
+        "application/pdf",
+        cast(list[str], []),
+    )
+    PNG = (
+        "png",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".png"],
+        "image/png",
+        cast(list[str], []),
+    )
+    PPT = (
+        "ppt",
+        "ppt",
+        ["pptx"],
+        "ppt",
+        [".ppt"],
+        "application/vnd.ms-powerpoint",
+        cast(list[str], []),
+    )
+    PPTX = (
+        "pptx",
+        "pptx",
+        ["pptx"],
+        "pptx",
+        [".pptx"],
+        "application/vnd.openxmlformats-officedocument.presentationml.presentation",
+        cast(list[str], []),
+    )
+    RST = ("rst", "rst", ["pypandoc"], "rst", [".rst"], "text/x-rst", cast(list[str], []))
+    RTF = ("rtf", "rtf", ["pypandoc"], "rtf", [".rtf"], "text/rtf", ["application/rtf"])
+    TIFF = (
+        "tiff",
+        "image",
+        ["unstructured_inference"],
+        "image",
+        [".tiff"],
+        "image/tiff",
+        cast(list[str], []),
+    )
+    TSV = ("tsv", "tsv", ["pandas"], "tsv", [".tab", ".tsv"], "text/tsv", cast(list[str], []))
+    TXT = (
+        "txt",
+        "text",
+        cast(list[str], []),
+        None,
+        [
+            ".txt",
+            ".text",
+            # NOTE(robinson) - for now we are treating code files as plain text
+            ".c",
+            ".cc",
+            ".cpp",
+            ".cs",
+            ".cxx",
+            ".go",
+            ".java",
+            ".js",
+            ".log",
+            ".php",
+            ".py",
+            ".rb",
+            ".swift",
+            ".ts",
+            ".yaml",
+            ".yml",
+        ],
+        "text/plain",
+        [
+            # NOTE(robinson) - In the future, we may have special processing for YAML files
+            # instead of treating them as plaintext.
+            "text/yaml",
+            "application/x-yaml",
+            "application/yaml",
+            "text/x-yaml",
+        ],
+    )
+    WAV = (
+        "wav",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".wav"],
+        "audio/wav",
+        [
+            "audio/vnd.wav",
+            "audio/vnd.wave",
+            "audio/wave",
+            "audio/x-pn-wav",
+            "audio/x-wav",
+        ],
+    )
+    WEBM = (
+        "webm",
+        "audio",
+        cast(list[str], []),  # STT agent deps validated at runtime by the chosen agent
+        "audio",
+        [".webm"],
+        "audio/webm",
+        [],  # Do not alias video/webm: WebM is a container; this type is for audio-only.
+    )
+    XLS = (
+        "xls",
+        "xlsx",
+        ["pandas", "openpyxl"],
+        "xlsx",
+        [".xls"],
+        "application/vnd.ms-excel",
+        cast(list[str], []),
+    )
+    XLSX = (
+        "xlsx",
+        "xlsx",
+        ["pandas", "openpyxl"],
+        "xlsx",
+        [".xlsx"],
+        "application/vnd.openxmlformats-officedocument.spreadsheetml.sheet",
+        cast(list[str], []),
+    )
+    XML = ("xml", "xml", cast(list[str], []), None, [".xml"], "application/xml", ["text/xml"])
+    ZIP = ("zip", None, cast(list[str], []), None, [".zip"], "application/zip", cast(list[str], []))
+
+    UNK = (
+        "unk",
+        None,
+        cast(list[str], []),
+        None,
+        cast(list[str], []),
+        "application/octet-stream",
+        cast(list[str], []),
+    )
+    EMPTY = (
+        "empty",
+        None,
+        cast(list[str], []),
+        None,
+        cast(list[str], []),
+        "inode/x-empty",
+        cast(list[str], []),
+    )
+
+
+def create_file_type(
+    name: str,
+    *,
+    canonical_mime_type: str,
+    importable_package_dependencies: Iterable[str] | None = None,
+    extra_name: str | None = None,
+    extensions: Iterable[str] | None = None,
+    alias_mime_types: Iterable[str] | None = None,
+) -> FileType:
+    """Register a new FileType member."""
+    type_ = _create_file_type_enum(
+        FileType,
+        name,
+        None,
+        importable_package_dependencies or cast(list[str], []),
+        extra_name,
+        extensions or cast(list[str], []),
+        canonical_mime_type,
+        alias_mime_types or cast(list[str], []),
+        None,
+    )
+    type_._name_ = name
+    FileType._member_map_[name] = type_
+    return type_
+
+
+_P = ParamSpec("_P")
+
+
+def register_partitioner(
+    file_type: FileType,
+) -> Callable[[Callable[_P, list[Element]]], Callable[_P, list[Element]]]:
+    def decorator(func: Callable[_P, list[Element]]) -> Callable[_P, list[Element]]:
+        file_type._partitioner_full_module_path = func.__module__ + "." + func.__name__
+        return func
+
+    return decorator

unstructured/file_utils/ndjson.py

@@ -0,0 +1,67 @@
+"""
+Adds support for working with newline-delimited JSON (ndjson) files. This format is useful for
+streaming json content that would otherwise not be possible using raw JSON files.
+"""
+
+import json
+from typing import IO, Any
+
+
+def dumps(obj: list[dict[str, Any]], **kwargs) -> str:
+    """
+    Converts the list of dictionaries into string representation
+
+    Args:
+        obj (list[dict[str, Any]]): List of dictionaries to convert
+        **kwargs: Additional keyword arguments to pass to json.dumps
+
+    Returns:
+        str: string representation of the list of dictionaries
+    """
+    return "\n".join(json.dumps(each, **kwargs) for each in obj)
+
+
+def dump(obj: list[dict[str, Any]], fp: IO, **kwargs) -> None:
+    """
+    Writes the list of dictionaries to a newline-delimited file
+
+    Args:
+        obj (list[dict[str, Any]]): List of dictionaries to convert
+        fp (IO): File pointer to write the string representation to
+        **kwargs: Additional keyword arguments to pass to json.dumps
+
+    Returns:
+        None
+    """
+    # Indent breaks ndjson formatting
+    kwargs["indent"] = None
+    text = dumps(obj, **kwargs)
+    fp.write(text)
+
+
+def loads(s: str, **kwargs) -> list[dict[str, Any]]:
+    """
+    Converts the raw string into a list of dictionaries
+
+    Args:
+        s (str): Raw string to convert
+        **kwargs: Additional keyword arguments to pass to json.loads
+
+    Returns:
+        list[dict[str, Any]]: List of dictionaries parsed from the input string
+    """
+    return [json.loads(line, **kwargs) for line in s.splitlines()]
+
+
+def load(fp: IO, **kwargs) -> list[dict[str, Any]]:
+    """
+    Converts the contents of the file into a list of dictionaries
+
+    Args:
+        fp (IO): File pointer to read the string representation from
+        **kwargs: Additional keyword arguments to pass to json.loads
+
+    Returns:
+        list[dict[str, Any]]: List of dictionaries parsed from the file
+    """
+    return loads(fp.read(), **kwargs)