chore: import upstream snapshot with attribution

scripts/airtable-test-helpers/component_ids.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Components below are all created using Airtable UI, however, in case they need
+# to be recreated, it is also possible to create them using the Web API.
+# pyairtable does not yet support creating these components (bases, tables).
+
+# For documentation on the Web API for creating bases, check:
+# https://airtable.com/developers/web/api/create-base
+
+# For creating lots of tables inside a base, check:
+# create_scale_test_components.sh
+
+LARGE_TABLE_BASE_ID="appQqieVsbxpwwD3i"
+LARGE_TABLE_TABLE_ID="tbll85GCfxED1OrvC"
+LARGE_BASE_BASE_ID="appjPRwoyawsapoGW"
+LARGE_WORKSPACE_BASE_ID_1="appSSCNWuIMjzeraO"
+LARGE_WORKSPACE_BASE_ID_2="appyvCsaHWn38RzFc"
+LARGE_WORKSPACE_BASE_ID_3="appbd8fkBv3AXj0Ab"
+LARGE_WORKSPACE_BASE_ID_4="appHEvCPnpfiAwjPE"
+LARGE_WORKSPACE_BASE_ID_5="appL9ND7LVWaItAmC"
+LARGE_WORKSPACE_BASE_ID_6="appOGnidMsh93yCQI"
+LARGE_WORKSPACE_BASE_ID_7="apps71HjvZRRgqHkz"
+LARGE_WORKSPACE_BASE_ID_8="appvDbw5f7jCQqdsr"
+LARGE_WORKSPACE_BASE_ID_9="appGFdtbLmqf2k8Ly"
+LARGE_WORKSPACE_BASE_ID_10="appTn61bfU8vCIkGf"
+LARGE_WORKSPACE_BASE_ID_11="app1c4CtIQ4ZToHIR"
+LARGE_WORKSPACE_BASE_ID_12="apphvDFg6OC7l1xwo"
+# shellcheck disable=SC2034
+LARGE_TEST_LIST_OF_PATHS="$LARGE_BASE_BASE_ID $LARGE_TABLE_BASE_ID $LARGE_WORKSPACE_BASE_ID_1 $LARGE_WORKSPACE_BASE_ID_2 $LARGE_WORKSPACE_BASE_ID_3 $LARGE_WORKSPACE_BASE_ID_4 $LARGE_WORKSPACE_BASE_ID_5 $LARGE_WORKSPACE_BASE_ID_6 $LARGE_WORKSPACE_BASE_ID_7 $LARGE_WORKSPACE_BASE_ID_8 $LARGE_WORKSPACE_BASE_ID_9 $LARGE_WORKSPACE_BASE_ID_10 $LARGE_WORKSPACE_BASE_ID_11 $LARGE_WORKSPACE_BASE_ID_12"
+
+export LARGE_TABLE_BASE_ID
+export LARGE_TABLE_TABLE_ID

scripts/airtable-test-helpers/create_scale_test_components.py

@@ -0,0 +1,82 @@
+import os
+
+# import pyairtable as pyair
+from pyairtable import Api
+
+from unstructured.logger import logger
+
+SCALE_TEST_NUMBER_OF_RECORDS = 20_000
+
+# Access token that has read and write permissions for the respective workspace
+token = os.environ["AIRTABLE_ACCESS_TOKEN_WRITE"]
+
+# You can find the IDs below defined in component_ids.sh
+# In case new ones are needed to be created, there's guidance below and in component_ids.sh.
+
+# ID of a new base that is intended to contain one large table.
+# The table will be filled using this python script.
+# If the ID is not in the environment, it is possible to create a new base
+# via the Airtable UI, and get the base ID from the URL structure.
+# (https://support.airtable.com/docs/finding-airtable-ids)
+large_table_base_id = os.environ["LARGE_TABLE_BASE_ID"]
+
+# ID of the one table inside the base "large_table_base".
+# The table is intended to be large, and will be filled using this python script.
+# If the ID is not in the environment, it is possible to create a new table
+# via the Airtable UI, and get the table ID from the URL structure.
+# (https://support.airtable.com/docs/finding-airtable-ids)
+large_table_table_id = os.environ["LARGE_TABLE_TABLE_ID"]
+
+# ID of a base that is intended to contain lots of tables.
+# large_base_base_id = os.environ["LARGE_BASE_BASE_ID"]
+# Creating tables is not yet supported in pyairtable. Try Airtable Web API instead:
+# https://airtable.com/developers/web/api/create-base"
+
+
+def create_n_bases(api, number_of_bases):
+    raise NotImplementedError(
+        "Creating bases is not yet supported in pyairtable. \
+                              Try Airtable Web API instead: \
+                              https://airtable.com/developers/web/api/create-base",
+    )
+    # if len(pyair.metadata.get_api_bases(api)["bases"])>99:
+    #     logger.warning("Airtable Org already has a high number of bases. \
+    #                Aborting creation of new bases to avoid duplication and bloating.")
+    #     return
+
+    number_of_bases
+
+
+def create_n_tables(base, number_of_tables):
+    raise NotImplementedError(
+        "Creating tables is not yet supported in pyairtable. \
+                              Try Airtable Web API instead: \
+                              https://airtable.com/developers/web/api/create-table",
+    )
+    # if len(pyair.metadata.get_base_schema(base)["tables"])>99:
+    #     logger.warning("Base already has a high number of tables. \
+    #               Aborting creation of new tables to avoid duplication and bloating.")
+    #     return
+
+
+def create_n_records(table, number_of_records):
+    logger.warning(
+        "Fetching table records to count, before creation of new records.\
+                   This should take around 1 second per 415 records.",
+    )
+    if len(table.all()) > SCALE_TEST_NUMBER_OF_RECORDS - 1:
+        logger.warning(
+            "Table already has a high number of records. \
+                Aborting creation of new records to avoid duplication and bloating.",
+        )
+        return
+
+    records = [{"Name": f"My Name is {i}"} for i in range(number_of_records)]
+    table.batch_create(records)
+
+
+if __name__ == "__main__":
+    api = Api(token)
+    large_table = api.table(large_table_base_id, large_table_table_id)
+    logger.info("Creating records, this should take about 1 second per 40 records.")
+    create_n_records(large_table, SCALE_TEST_NUMBER_OF_RECORDS)

scripts/airtable-test-helpers/create_scale_test_components.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# This scripts creates a large number of tables inside an Airtable base.
+
+# shellcheck disable=SC2001,SC1091
+source ./scripts/airtable-test-helpers/component_ids.sh
+
+base_data='{"description": "Table-X of the test tables for the test LARGE_BASE.", "fields": [{"description": "Name of the row","name": "Name","type": "singleLineText"}],"name": "LARGE_BASE_TABLE_X"}'
+for ((i = 1; i <= 100; i++)); do
+  item="$(echo "$base_data" | sed "s/X/$i/g")"
+  curl -X POST "https://api.airtable.com/v0/meta/bases/$LARGE_BASE_BASE_ID/tables" \
+    -H "Authorization: Bearer $AIRTABLE_ACCESS_TOKEN_WRITE2" \
+    -H "Content-Type: application/json" \
+    --data "$item"
+done

scripts/airtable-test-helpers/print_num_rows_df.py

@@ -0,0 +1,28 @@
+import argparse
+import json
+from io import StringIO
+
+import pandas as pd
+
+
+def number_of_rows(file_path):
+    with open(file_path) as file:
+        data = json.load(file)
+        df = pd.read_csv(StringIO(data[0]["text"]))
+    return len(df)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Read Unstructured Ingest output file and print the number of rows",
+    )
+
+    parser.add_argument(
+        "--structured-output-file-path",
+        help="Path to Unstructured Ingest output file",
+    )
+
+    args = parser.parse_args()
+
+    output_path = args.structured_output_file_path
+    print(number_of_rows(output_path))

scripts/check-licenses.sh

@@ -0,0 +1,77 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Allowed license families (partial-match against the License metadata field).
+# Covers the standard permissive + weak-copyleft licenses the project accepts.
+# Build the semicolon-separated allowlist for --partial-match.
+# Each entry is matched as a case-insensitive substring against the package's
+# License metadata field. Order does not matter.
+ALLOWED="Apache;\
+BSD;\
+MIT;\
+ISC;\
+MPL;\
+Mozilla;\
+LGPL;\
+GNU Lesser General Public License;\
+GNU Library or Lesser General Public License;\
+GNU General Public License v2;\
+PSF;\
+Python Software Foundation;\
+Unlicense;\
+HPND;\
+Historical Permission Notice and Disclaimer;\
+CNRI-Python;\
+Python-2.0"
+
+# Packages whose license metadata is missing, non-standard, or proprietary but
+# known-good for this project. Each has been manually verified against the
+# upstream source repository.
+IGNORED_PACKAGES=(
+  # Metadata missing -- verified permissive on GitHub
+  arro3-core        # MIT / Apache-2.0 (geoarrow/geoarrow-rs)
+  chroma-hnswlib    # Apache-2.0 (chroma-core/hnswlib)
+  google-crc32c     # Apache-2.0 (googleapis/python-crc32c)
+  iopath            # MIT (facebookresearch/iopath)
+  pypdfium2         # BSD-3-Clause (PDFium/PDFium)
+  sentencepiece     # Apache-2.0 (google/sentencepiece)
+  voyageai          # MIT (voyage-ai/voyageai-python)
+  matplotlib-inline # BSD 3-Clause (ipython/matplotlib-inline)
+
+  # Permissive but non-standard classifier
+  lmdb # OpenLDAP Public License (BSD-style, jnwatson/py-lmdb)
+  pykx # KDB+ proprietary (KxSystems/pykx, transitive dep of kdbai-client)
+
+  # NVIDIA CUDA runtime libs (proprietary, torch transitive dependencies)
+  cuda-bindings
+  nvidia-cublas-cu12
+  nvidia-cuda-cupti-cu12
+  nvidia-cuda-nvrtc-cu12
+  nvidia-cuda-runtime-cu12
+  nvidia-cudnn-cu12
+  nvidia-cufft-cu12
+  nvidia-cufile-cu12
+  nvidia-curand-cu12
+  nvidia-cusolver-cu12
+  nvidia-cusparse-cu12
+  nvidia-cusparselt-cu12
+  nvidia-nccl-cu12
+  nvidia-nvjitlink-cu12
+  nvidia-nvshmem-cu12
+  nvidia-nvtx-cu12
+)
+
+echo "Checking licenses for installed packages..."
+uv run pip-licenses \
+  --partial-match \
+  --allow-only="$ALLOWED" \
+  --ignore-packages "${IGNORED_PACKAGES[@]}"
+EXIT_CODE=$?
+
+if [ "$EXIT_CODE" -eq 0 ]; then
+  echo "All dependencies have authorized licenses."
+else
+  echo "There are dependencies with unauthorized or unknown licenses."
+  exit 1
+fi

scripts/check-new-release-version.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+set -eux
+
+# Function to check if the current version is a non-dev version
+function is_non_dev_version {
+  local VERSION="$1"
+  [[ "$VERSION" != *"-dev"* ]]
+}
+
+# Function to get the version from the current main branch
+function get_main_branch_version {
+  local VERSION
+  git fetch origin main
+  VERSION=$(git show origin/main:unstructured/__version__.py | grep -o -m 1 -E "(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-[a-zA-Z0-9.-]+)?")
+  echo "$VERSION"
+}
+
+# Get the current version from the file
+CURRENT_VERSION=$(grep -o -m 1 -E "(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-dev[0-9]+)?" "unstructured/__version__.py")
+
+# Check if the current version is a non-dev version and not matching the main version
+if is_non_dev_version "$CURRENT_VERSION" && [ "$(get_main_branch_version)" != "$CURRENT_VERSION" ]; then
+  echo "New release version: $CURRENT_VERSION"
+fi

scripts/chroma-test-helpers/create-and-check-chroma.sh

@@ -0,0 +1,6 @@
+#!/usr/bin/env bash
+
+set -e
+
+# $1 is the path for chroma to write the contents to. The symbol "&" runs process in background
+python "$VIRTUAL_ENV/bin/chroma" run --path "$1" &

scripts/collect_env.py

@@ -0,0 +1,242 @@
+import platform
+import shutil
+import subprocess
+
+import pkg_resources
+
+from unstructured.utils import dependency_exists
+
+
+def command_exists(command):
+    """
+    Check if a command exists in the system
+
+    Args:
+        command (str): The command to check
+
+    Returns:
+        bool: True if command exists, False otherwise
+    """
+    return shutil.which(command) is not None
+
+
+def get_python_version():
+    """
+    Get the current Python version
+
+    Returns:
+        str: The current Python version
+    """
+    return platform.python_version()
+
+
+def get_os_version():
+    """
+    Get the current operating system version
+
+    Returns:
+        str: The current operating system version
+    """
+    return platform.platform()
+
+
+def is_python_package_installed(package_name: str):
+    """
+    Check if a Python package is installed
+
+    Args:
+        package_name (str): The Python package to check
+
+    Returns:
+        bool: True if package is installed, False otherwise
+    """
+    result = subprocess.run(
+        ["pip", "list"],
+        stdout=subprocess.PIPE,
+        text=True,
+        check=True,
+    )
+
+    return any(line.lower().startswith(package_name.lower()) for line in result.stdout.splitlines())
+
+
+def is_brew_package_installed(package_name: str):
+    """
+    Check if a Homebrew package is installed
+
+    Args:
+        package_name (str): The package to check
+
+    Returns:
+        bool: True if package is installed, False otherwise
+    """
+    if not command_exists("brew"):
+        return False
+
+    result = subprocess.run(
+        ["brew", "list"],
+        stdout=subprocess.PIPE,
+        text=True,
+        check=True,
+    )
+
+    for line in result.stdout.splitlines():
+        if line.lower().startswith(package_name.lower()):
+            return True
+
+    result = subprocess.run(
+        ["brew", "list", "--cask"],
+        stdout=subprocess.PIPE,
+        text=True,
+        check=True,
+    )
+
+    return any(line.lower().startswith(package_name.lower()) for line in result.stdout.splitlines())
+
+
+def get_python_package_version(package_name):
+    """
+    Get the version of a Python package
+
+    Args:
+        package_name (str): The Python package to check
+
+    Returns:
+        str: Version of the package, None if package is not installed
+    """
+    try:
+        return pkg_resources.get_distribution(package_name).version
+    except pkg_resources.DistributionNotFound:
+        return None
+
+
+def get_brew_package_version(package_name):
+    """
+    Get the version of a Homebrew package
+
+    Args:
+        package_name (str): The package to check
+
+    Returns:
+        str: Version of the package, None if package is not installed
+    """
+    if not command_exists("brew"):
+        return None
+
+    result = subprocess.run(
+        ["brew", "info", package_name],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.DEVNULL,
+        text=True,
+    )
+
+    for line in result.stdout.splitlines():
+        return line
+
+    return None
+
+
+def get_libmagic_version():
+    """
+    Get the version of libmagic
+
+    Returns:
+        str: Version of libmagic, None if libmagic is not installed
+    """
+    result = subprocess.run(
+        ["file", "--version", "--headless"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+
+    return result.stdout.strip()
+
+
+def get_libreoffice_version():
+    """
+    Get the version of LibreOffice
+
+    Returns:
+        str: Version of LibreOffice, None if LibreOffice is not installed
+    """
+    result = subprocess.run(
+        ["libreoffice", "--version", "--headless"],
+        stdout=subprocess.PIPE,
+        stderr=subprocess.PIPE,
+        text=True,
+    )
+
+    return result.stdout.strip()
+
+
+def main():
+    """
+    The main function to run all checks
+    """
+    print("OS version: ", get_os_version())
+    print("Python version: ", get_python_version())
+
+    if dependency_exists("unstructured"):
+        print("unstructured version: ", get_python_package_version("unstructured"))
+    else:
+        print("unstructured is not installed")
+
+    if dependency_exists("unstructured_inference"):
+        print(
+            "unstructured-inference version: ",
+            get_python_package_version("unstructured-inference"),
+        )
+    else:
+        print("unstructured-inference is not installed")
+
+    if dependency_exists("pytesseract"):
+        print(
+            "pytesseract version: ",
+            get_python_package_version("pytesseract"),
+        )
+    else:
+        print("pytesseract is not installed")
+
+    if dependency_exists("torch"):
+        print("Torch version: ", get_python_package_version("torch"))
+    else:
+        print("Torch is not installed")
+
+    if dependency_exists("detectron2"):
+        print("Detectron2 version: ", get_python_package_version("detectron2"))
+    else:
+        print("Detectron2 is not installed")
+
+    if is_python_package_installed("paddlepaddle") or is_python_package_installed(
+        "paddleocr",
+    ):
+        print(
+            "PaddleOCR version: ",
+            get_python_package_version("paddlepaddle") or get_python_package_version("paddleocr"),
+        )
+    else:
+        print("PaddleOCR is not installed")
+
+    if is_brew_package_installed("libmagic"):
+        print("Libmagic version: ", get_brew_package_version("libmagic"))
+    else:
+        libmagic_version = get_libmagic_version()
+        if libmagic_version:
+            print(f"Libmagic version: {libmagic_version}")
+        else:
+            print("Libmagic is not installed")
+
+    if platform.system() != "Windows":
+        if is_brew_package_installed("libreoffice"):
+            print("LibreOffice version: ", get_brew_package_version("libreoffice"))
+        else:
+            libreoffice_version = get_libreoffice_version()
+            if libreoffice_version:
+                print("LibreOffice version: ", libreoffice_version)
+            else:
+                print("LibreOffice is not installed")
+
+
+if __name__ == "__main__":
+    main()

scripts/convert/elements_json_to_format.py

@@ -0,0 +1,99 @@
+import argparse
+import logging
+import os
+from pathlib import Path
+
+from unstructured.partition.html.convert import elements_to_html
+from unstructured.staging.base import elements_from_json, elements_to_md
+
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def json_to_format(
+    filepath: Path,
+    outdir: Path,
+    format_type: str,
+    exclude_binary_image_data: bool,
+    no_group_by_page: bool,
+):
+    logger.info("Processing: %s", filepath)
+    elements = elements_from_json(str(filepath))
+
+    if format_type == "html":
+        output_content = elements_to_html(elements, exclude_binary_image_data, no_group_by_page)
+        file_extension = ".html"
+    elif format_type == "markdown":
+        output_content = elements_to_md(
+            elements, exclude_binary_image_data=exclude_binary_image_data
+        )
+        file_extension = ".md"
+    else:
+        raise ValueError(f"Unsupported format: {format_type}. Supported formats: html, markdown")
+
+    outpath = outdir / filepath.with_suffix(file_extension).name
+    os.makedirs(outpath.parent, exist_ok=True)
+    with open(outpath, "w+") as f:
+        f.write(output_content)
+    logger.info(f"{format_type.upper()} rendered and saved to: %s", outpath)
+
+
+def multiple_json_to_format(
+    path: Path,
+    outdir: Path,
+    format_type: str,
+    exclude_binary_image_data: bool,
+    no_group_by_page: bool,
+):
+    for root, _, files in os.walk(path):
+        for file in files:
+            if file.endswith(".json"):
+                json_file_path = Path(root) / file
+                outpath = outdir / json_file_path.relative_to(path).parent
+                json_to_format(
+                    json_file_path,
+                    outpath,
+                    format_type,
+                    exclude_binary_image_data,
+                    no_group_by_page,
+                )
+
+
+def main():
+    parser = argparse.ArgumentParser(description="Convert JSON elements to HTML or Markdown.")
+    parser.add_argument(
+        "filepath",
+        type=str,
+        help="""Path to the JSON file or directory containing elements.
+        If given directory it will convert all JSON files in directory
+        and all sub-directories.""",
+    )
+    parser.add_argument(
+        "--outdir", type=str, help="Output directory for the output file.", default=""
+    )
+    parser.add_argument(
+        "--format",
+        type=str,
+        choices=["html", "markdown"],
+        default="html",
+        help="Output format: html or markdown (default: html)",
+    )
+    parser.add_argument(
+        "--exclude-img", action="store_true", help="Exclude binary image data from the output."
+    )
+    parser.add_argument(
+        "--no-group", action="store_true", help="Don't group elements by pages (HTML only)."
+    )
+    args = parser.parse_args()
+
+    filepath = Path(args.filepath)
+    outdir = Path(args.outdir)
+
+    if filepath.is_file():
+        json_to_format(filepath, outdir, args.format, args.exclude_img, args.no_group)
+    else:
+        multiple_json_to_format(filepath, outdir, args.format, args.exclude_img, args.no_group)
+
+
+if __name__ == "__main__":
+    main()

scripts/convert/rendered_html_from_elements.py

@@ -0,0 +1,92 @@
+# pyright: reportPrivateUsage=false
+
+"""
+Script to render HTML from unstructured elements.
+NOTE: This script is not intended to be used as a module.
+NOTE: For now script is only intended to be used with elements generated with
+      `partition_html(html_parser_version=v2)`
+TODO: It was noted that unstructured_elements_to_ontology func always returns a single page
+      This script is using helper functions to handle multiple pages.
+"""
+
+import argparse
+import html
+import logging
+import os
+import select
+import sys
+
+from unstructured.partition.html.transformations import unstructured_elements_to_ontology
+from unstructured.staging.base import elements_from_json
+
+# Configure logging
+logging.basicConfig(level=logging.INFO, format="%(asctime)s - %(levelname)s - %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def rendered_html(*, filepath: str | None = None, text: str | None = None) -> str:
+    """Renders HTML from a JSON file with unstructured elements.
+
+    Args:
+        filepath (str): path to JSON file with unstructured elements.
+
+    Returns:
+        str: HTML content.
+    """
+    if filepath is None and text is None:
+        logger.error("Either filepath or text must be provided.")
+        raise ValueError("Either filepath or text must be provided.")
+    if filepath is not None and text is not None:
+        logger.error("Both filepath and text cannot be provided.")
+        raise ValueError("Both filepath and text cannot be provided.")
+    if filepath is not None:
+        logger.info("Rendering HTML from file: %s", filepath)
+    else:
+        logger.info("Rendering HTML from text.")
+
+    unstructured_elements = elements_from_json(filename=filepath, text=text)
+    ontology_root = unstructured_elements_to_ontology(unstructured_elements)
+    html_document = ontology_root.to_html()
+    unescaped_html = html.unescape(html_document)
+    return unescaped_html
+
+
+def _main():
+    if os.getenv("PROCESS_FROM_STDIN") == "true":
+        logger.info("Processing from STDIN (PROCESS_FROM_STDIN is set to 'true')")
+        if select.select([sys.stdin], [], [], 0.1)[0]:
+            content = sys.stdin.read()
+            html = rendered_html(text=content)
+            sys.stdout.write(html)
+        else:
+            logger.error("No input provided via STDIN. Exiting.")
+            sys.exit(1)
+    else:
+        logger.info("Processing from command line arguments")
+        parser = argparse.ArgumentParser(description="Render HTML from unstructured elements.")
+        parser.add_argument(
+            "filepath", help="Path to JSON file with unstructured elements.", type=str
+        )
+        parser.add_argument(
+            "--outdir",
+            help="Path to directory where the rendered html will be stored.",
+            type=str,
+            default=None,
+            nargs="?",
+        )
+        args = parser.parse_args()
+
+        html = rendered_html(filepath=args.filepath)
+        if args.outdir is None:
+            args.outdir = os.path.dirname(args.filepath)
+        os.makedirs(args.outdir, exist_ok=True)
+        outpath = os.path.join(
+            args.outdir, os.path.basename(args.filepath).replace(".json", ".html")
+        )
+        with open(outpath, "w") as f:
+            f.write(html)
+        logger.info("HTML rendered and saved to: %s", outpath)
+
+
+if __name__ == "__main__":
+    _main()

scripts/docker-build-ubuntu.sh

@@ -0,0 +1,11 @@
+#!/usr/bin/env bash
+
+# Mainly useful for building an image from which to update test-ingest fixtures
+
+set -eu -o pipefail
+
+# Change to the root of the repository
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+cd "$SCRIPT_DIR"/.. || exit 1
+
+docker build -t unstructured-ubuntu:latest --progress plain -f docker/ubuntu-22/Dockerfile .

scripts/docker-build.sh

@@ -0,0 +1,18 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+DOCKER_REPOSITORY="${DOCKER_REPOSITORY:-quay.io/unstructured-io/unstructured}"
+DOCKER_IMAGE="${DOCKER_IMAGE:-unstructured:dev}"
+
+DOCKER_BUILD_CMD=(docker buildx build --load -f Dockerfile
+  --build-arg BUILDKIT_INLINE_CACHE=1
+  --progress plain
+  --cache-from "$DOCKER_REPOSITORY":latest
+  -t "$DOCKER_IMAGE" .)
+
+# only build for specific platform if DOCKER_BUILD_PLATFORM is set
+if [ -n "${DOCKER_BUILD_PLATFORM:-}" ]; then
+  DOCKER_BUILD_CMD+=("--platform=$DOCKER_BUILD_PLATFORM")
+fi
+
+DOCKER_BUILDKIT=1 "${DOCKER_BUILD_CMD[@]}"

scripts/docker-smoke-test.sh

@@ -0,0 +1,47 @@
+#!/usr/bin/env bash
+
+# Start the containerized repository and run ingest tests
+
+# shellcheck disable=SC2317  # Shellcheck complains that trap functions are unreachable...
+# shellcheck disable=SC2329  # Functions are invoked indirectly
+
+set -eux -o pipefail
+
+CONTAINER_NAME=unstructured-smoke-test
+DOCKER_IMAGE="${DOCKER_IMAGE:-unstructured:dev}"
+
+# Change to the root of the repository
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+cd "$SCRIPT_DIR"/.. || exit 1
+
+start_container() {
+  echo Starting container "$CONTAINER_NAME"
+  docker run -dt --rm --name "$CONTAINER_NAME" "$DOCKER_IMAGE"
+}
+
+await_container() {
+  echo Waiting for container to start
+  until [ "$(docker inspect -f '{{.State.Status}}' $CONTAINER_NAME)" == "running" ]; do
+    sleep 1
+  done
+}
+
+stop_container() {
+  echo Stopping container "$CONTAINER_NAME"
+  docker stop "$CONTAINER_NAME"
+}
+
+start_container
+
+# Regardless of test result, stop the container
+trap stop_container EXIT
+
+await_container
+
+# Run the tests
+docker cp test_unstructured_ingest $CONTAINER_NAME:/app
+docker exec -u root "$CONTAINER_NAME" /bin/bash -c "chown -R notebook-user:notebook-user /app/test_unstructured_ingest"
+docker exec "$CONTAINER_NAME" /bin/bash -c "/app/test_unstructured_ingest/src/local.sh"
+
+result=$?
+exit $result

scripts/elasticsearch-test-helpers/common/docker-compose.yaml

@@ -0,0 +1,17 @@
+services:
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:8.19.10
+    container_name: es-test
+    ports:
+      - 9200:9200
+      - 9300:9300
+    environment:
+      - xpack.security.enabled=true
+      - discovery.type=single-node
+      - ELASTIC_PASSWORD=${ELASTIC_PASSWORD}
+      - ELASTIC_USER=${ELASTIC_USER}
+    healthcheck:
+      test: ["CMD-SHELL", "curl --silent --fail -u ${ELASTIC_USER}:${ELASTIC_PASSWORD} localhost:9200/_cluster/health || exit 1"]
+      interval: 10s
+      timeout: 30s
+      retries: 3

scripts/elasticsearch-test-helpers/common/es-dest-ingest-test-creds.env

@@ -0,0 +1,6 @@
+# These credentials are for the ES index within ingest test ES docker image,
+# which is stopped immediately after ingest ES destination connector test completes.
+# Do not use these credentials for any other purpose than local development, or testing.
+# Do not use these credentials for any permanent / long life ES cluster; be it in dev or prod.
+export ELASTIC_USER=elastic
+export ELASTIC_PASSWORD=Vth0Zd0wxme

scripts/elasticsearch-test-helpers/destination_connector/create-elasticsearch-instance.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(dirname "$(dirname "$(realpath "$0")")")
+ENV_FILE="$SCRIPT_DIR"/common/es-dest-ingest-test-creds.env
+
+# Create the Elasticsearch cluster
+docker compose version
+docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR"/common/docker-compose.yaml up --wait
+docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR"/common/docker-compose.yaml ps
+
+echo "Cluster is live."
+python "$SCRIPT_DIR"/destination_connector/create_index.py

scripts/elasticsearch-test-helpers/destination_connector/create_index.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+from elasticsearch import Elasticsearch
+from es_cluster_config import (
+    CLUSTER_URL,
+    INDEX_NAME,
+    PASSWORD,
+    USER,
+    mappings,
+)
+
+print("Connecting to the Elasticsearch cluster.")
+es = Elasticsearch(CLUSTER_URL, basic_auth=(USER, PASSWORD), request_timeout=30)
+print(f"{es.info()}")
+
+print("Creating an Elasticsearch index for testing ingest elasticsearch destination connector.")
+response = es.options(max_retries=5).indices.create(index=INDEX_NAME, mappings=mappings)
+if response.meta.status != 200:
+    raise RuntimeError("failed to create index")
+
+es.indices.refresh(index=INDEX_NAME)
+response = es.cat.count(index=INDEX_NAME, format="json")
+
+print("Succesfully created an Elasticsearch index for testing elasticsearch ingest.")

scripts/elasticsearch-test-helpers/destination_connector/elasticsearch_elements_mappings.json

@@ -0,0 +1,144 @@
+{
+   "properties": {
+      "element_id": {
+         "type": "keyword"
+      },
+      "text": {
+         "type": "text",
+         "analyzer": "english"
+      },
+      "type": {
+         "type": "text"
+      },
+      "embeddings": {
+         "type": "dense_vector",
+         "dims": 384,
+         "index": "true",
+         "similarity": "cosine"
+      },
+      "metadata": {
+         "type": "object",
+         "properties": {
+            "category_depth": {
+               "type": "integer"
+            },
+            "parent_id": {
+               "type": "keyword"
+            },
+            "attached_to_filename": {
+               "type": "keyword"
+            },
+            "filetype": {
+               "type": "keyword"
+            },
+            "last_modified": {
+               "type": "date"
+            },
+            "file_directory": {
+               "type": "keyword"
+            },
+            "filename": {
+               "type": "keyword"
+            },
+            "data_source": {
+               "type": "object",
+               "properties": {
+                  "url": {
+                     "type": "text",
+                     "analyzer": "standard"
+                  },
+                  "version": {
+                     "type": "keyword"
+                  },
+                  "date_created": {
+                     "type": "date"
+                  },
+                  "date_modified": {
+                     "type": "date"
+                  },
+                  "date_processed": {
+                     "type": "date"
+                  },
+                  "record_locator": {
+                     "type": "keyword"
+                  },
+                  "permissions_data": {
+                     "type": "object"
+                  }
+               }
+            },
+            "coordinates": {
+               "type": "object",
+               "properties": {
+                  "system": {
+                     "type": "keyword"
+                  },
+                  "layout_width": {
+                     "type": "float"
+                  },
+                  "layout_height": {
+                     "type": "float"
+                  },
+                  "points": {
+                     "type": "float"
+                  }
+               }
+            },
+            "languages": {
+               "type": "keyword"
+            },
+            "page_number": {
+               "type": "integer"
+            },
+            "page_name": {
+               "type": "keyword"
+            },
+            "url": {
+               "type": "text",
+               "analyzer": "standard"
+            },
+            "links": {
+               "type": "object"
+            },
+            "link_urls": {
+               "type": "text"
+            },
+            "link_texts": {
+               "type": "text"
+            },
+            "sent_from": {
+               "type": "text",
+               "analyzer": "standard"
+            },
+            "sent_to": {
+               "type": "text",
+               "analyzer": "standard"
+            },
+            "subject": {
+               "type": "text",
+               "analyzer": "standard"
+            },
+            "section": {
+               "type": "text",
+               "analyzer": "standard"
+            },
+            "header_footer_type": {
+               "type": "keyword"
+            },
+            "emphasized_text_contents": {
+               "type": "text"
+            },
+            "emphasized_text_tags": {
+               "type": "keyword"
+            },
+            "text_as_html": {
+               "type": "text",
+               "analyzer": "standard"
+            },
+            "detection_class_prob": {
+               "type": "float"
+            }
+         }
+      }
+   }
+}

scripts/elasticsearch-test-helpers/destination_connector/es_cluster_config.py

@@ -0,0 +1,13 @@
+import json
+import os
+
+CLUSTER_URL = "http://localhost:9200"
+INDEX_NAME = "ingest-test-destination"
+USER = os.environ["ELASTIC_USER"]
+PASSWORD = os.environ["ELASTIC_PASSWORD"]
+MAPPING_PATH = (
+    "scripts/elasticsearch-test-helpers/destination_connector/elasticsearch_elements_mappings.json"
+)
+
+with open(MAPPING_PATH) as f:
+    mappings = json.load(f)

scripts/elasticsearch-test-helpers/destination_connector/test-ingest-elasticsearch-output.py

@@ -0,0 +1,133 @@
+#!/usr/bin/env python3
+
+from time import sleep, time
+from typing import List
+
+import click
+from elasticsearch import Elasticsearch
+from es_cluster_config import (
+    CLUSTER_URL,
+    INDEX_NAME,
+    PASSWORD,
+    USER,
+)
+
+from unstructured.embed.huggingface import HuggingFaceEmbeddingConfig, HuggingFaceEmbeddingEncoder
+
+
+def embeddings_for_text(text: str) -> List[float]:
+    embedding_encoder = HuggingFaceEmbeddingEncoder(config=HuggingFaceEmbeddingConfig())
+    return embedding_encoder.embed_query(text)
+
+
+def query(client: Elasticsearch, search_text: str):
+    # Query the index using the appropriate embedding vector for given query text
+    search_vector = embeddings_for_text(search_text)
+    # Constructing the search query
+    query = {
+        "query": {
+            "script_score": {
+                "query": {"match_all": {}},
+                "script": {
+                    "source": "cosineSimilarity(params.query_vector, 'embeddings') + 1.0",
+                    "params": {"query_vector": search_vector},
+                },
+            }
+        }
+    }
+    return client.search(index=INDEX_NAME, body=query)
+
+
+def validate_count(client: Elasticsearch, num_elements: int):
+    print(f"Validating that the count of documents in index {INDEX_NAME} is {num_elements}")
+    count = int(client.cat.count(index=INDEX_NAME, format="json")[0]["count"])
+    consistent = False
+    consistent_count = 1
+    desired_consistent_count = 5
+    timeout = 60
+    sleep_interval = 1
+    start_time = time()
+    print(f"initial count returned: {count}")
+    while not consistent and time() - start_time < timeout:
+        new_count = int(client.cat.count(index=INDEX_NAME, format="json")[0]["count"])
+        print(f"latest count returned: {new_count}")
+        if new_count == count:
+            consistent_count += 1
+        else:
+            count = new_count
+            consistent_count = 1
+        sleep(sleep_interval)
+        if consistent_count >= desired_consistent_count:
+            consistent = True
+    if not consistent:
+        raise TimeoutError(f"failed to get consistent count after {timeout}s")
+    assert count == num_elements, (
+        f"Elasticsearch dest check failed: got {count} items in index, "
+        f"expected {num_elements} items in index."
+    )
+    print(f"Elasticsearch destination test was successful with {count} items being uploaded.")
+
+
+def get_embeddings_len(client: Elasticsearch) -> int:
+    res = client.search(index=INDEX_NAME, size=1, query={"match_all": {}})
+    return len(res["hits"]["hits"][0]["_source"]["embeddings"])
+
+
+def validate_embeddings(client: Elasticsearch, embeddings: list[float]):
+    # Query the index using the appropriate embedding vector for given query text
+    # Verify that the top 1 result matches the expected chunk by checking the start text
+    print("Testing query to the embedded index.")
+    es_embeddings_len = get_embeddings_len(client=client)
+    assert len(embeddings) == es_embeddings_len, (
+        f"length of embeddings ({len(embeddings)}) doesn't "
+        f"match what exists in Elasticsearch ({es_embeddings_len})"
+    )
+    query_string = {
+        "field": "embeddings",
+        "query_vector": embeddings,
+        "k": 10,
+        "num_candidates": 10,
+    }
+    query_response = client.search(index=INDEX_NAME, knn=query_string)
+    response_found = query_response["hits"]["hits"][0]["_source"]
+    assert response_found["embeddings"] == embeddings
+    print("Query to the embedded index was successful and returned the expected result.")
+
+
+def validate(num_elements: int, embeddings: list[float]):
+    print(f"Checking contents of index {INDEX_NAME} at {CLUSTER_URL}")
+
+    print("Connecting to the Elasticsearch cluster.")
+    client = Elasticsearch(CLUSTER_URL, basic_auth=(USER, PASSWORD), request_timeout=30)
+    print(client.info())
+    validate_count(client=client, num_elements=num_elements)
+    validate_embeddings(client=client, embeddings=embeddings)
+
+
+def parse_embeddings(embeddings_str: str) -> list[float]:
+    if embeddings_str.startswith("["):
+        embeddings_str = embeddings_str[1:]
+    if embeddings_str.endswith("]"):
+        embeddings_str = embeddings_str[:-1]
+    embeddings_split = embeddings_str.split(",")
+    embeddings_split = [e.strip() for e in embeddings_split]
+    return [float(e) for e in embeddings_split]
+
+
+@click.command()
+@click.option(
+    "--num-elements", type=int, required=True, help="The expected number of elements to exist"
+)
+@click.option("--embeddings", type=str, required=True, help="List of embeddings to test")
+def run_validation(num_elements: int, embeddings: str):
+    try:
+        parsed_embeddings = parse_embeddings(embeddings_str=embeddings)
+    except ValueError as e:
+        raise TypeError(
+            f"failed to parse embeddings string into list of float: {embeddings}"
+        ) from e
+    validate(num_elements=num_elements, embeddings=parsed_embeddings)
+
+
+if __name__ == "__main__":
+    run_validation()

scripts/elasticsearch-test-helpers/source_connector/create-fill-and-check-es.sh

@@ -0,0 +1,14 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR="$(dirname "$(dirname "$(realpath "$0")")")"
+ENV_FILE="$SCRIPT_DIR"/common/es-dest-ingest-test-creds.env
+
+# Create the Elasticsearch cluster
+docker compose version
+docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR"/common/docker-compose.yaml up --wait
+docker compose --env-file "$ENV_FILE" -f "$SCRIPT_DIR"/common/docker-compose.yaml ps
+
+echo "Cluster is live."
+"$SCRIPT_DIR"/source_connector/create_and_fill_es.py

scripts/elasticsearch-test-helpers/source_connector/create_and_fill_es.py

@@ -0,0 +1,35 @@
+#!/usr/bin/env python3
+
+import pandas as pd
+from elasticsearch import Elasticsearch
+from elasticsearch.helpers import bulk
+from es_cluster_config import (
+    CLUSTER_URL,
+    DATA_PATH,
+    INDEX_NAME,
+    MAPPINGS,
+    PASSWORD,
+    USER,
+    form_elasticsearch_doc_dict,
+)
+
+print("Connecting to the Elasticsearch cluster.")
+es = Elasticsearch(CLUSTER_URL, basic_auth=(USER, PASSWORD), request_timeout=30)
+print(f"{es.info()}")
+df = pd.read_csv(DATA_PATH).dropna().reset_index()
+
+print("Creating an Elasticsearch index for testing elasticsearch ingest.")
+response = es.options(max_retries=5).indices.create(index=INDEX_NAME, mappings=MAPPINGS)
+if response.meta.status != 200:
+    raise RuntimeError("failed to create index")
+
+print("Loading data into the index.")
+bulk_data = []
+for i, row in df.iterrows():
+    bulk_data.append(form_elasticsearch_doc_dict(i, row))
+bulk(es, bulk_data)
+
+es.indices.refresh(index=INDEX_NAME)
+response = es.cat.count(index=INDEX_NAME, format="json")
+
+print("Successfully created and filled an Elasticsearch index for testing elasticsearch ingest.")

scripts/elasticsearch-test-helpers/source_connector/es_cluster_config.py

@@ -0,0 +1,37 @@
+import os
+
+DATA_PATH = "scripts/elasticsearch-test-helpers/source_connector/wiki_movie_plots_small.csv"
+CLUSTER_URL = "http://localhost:9200"
+INDEX_NAME = "movies"
+USER = os.environ["ELASTIC_USER"]
+PASSWORD = os.environ["ELASTIC_PASSWORD"]
+
+MAPPINGS = {
+    "properties": {
+        "title": {"type": "text", "analyzer": "english"},
+        "ethnicity": {"type": "text", "analyzer": "standard"},
+        "director": {"type": "text", "analyzer": "standard"},
+        "cast": {"type": "text", "analyzer": "standard"},
+        "genre": {"type": "text", "analyzer": "standard"},
+        "plot": {"type": "text", "analyzer": "english"},
+        "year": {"type": "integer"},
+        "wiki_page": {"type": "keyword"},
+    },
+}
+
+
+def form_elasticsearch_doc_dict(i, csv_row):
+    return {
+        "_index": INDEX_NAME,
+        "_id": i,
+        "_source": {
+            "title": csv_row["Title"],
+            "ethnicity": csv_row["Origin/Ethnicity"],
+            "director": csv_row["Director"],
+            "cast": csv_row["Cast"],
+            "genre": csv_row["Genre"],
+            "plot": csv_row["Plot"],
+            "year": csv_row["Release Year"],
+            "wiki_page": csv_row["Wiki Page"],
+        },
+    }

scripts/elasticsearch-test-helpers/source_connector/wiki_movie_plots_small.csv

@@ -0,0 +1,31 @@
+Release Year,Title,Origin/Ethnicity,Director,Cast,Genre,Wiki Page,Plot
+1901,Kansas Saloon Smashers,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Kansas_Saloon_Smashers,"A bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]"
+1901,Love by the Light of the Moon,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Love_by_the_Light_of_the_Moon,"The moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better."
+1901,The Martyred Presidents,American,Unknown,,unknown,https://en.wikipedia.org/wiki/The_Martyred_Presidents,"The film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination.
+In the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice."
+1901,"Terrible Teddy, the Grizzly King",American,Unknown,,unknown,"https://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_King","Lasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading ""His Photographer"" and ""His Press Agent"" respectively, follow him into the shot; the photographer sets up his camera. ""Teddy"" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. ""Teddy"" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. ""Teddy"" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs."
+1902,Jack and the Beanstalk,American,"George S. Fleming, Edwin S. Porter",,unknown,https://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film),"The earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince."
+1903,Alice in Wonderland,American,Cecil Hepworth,May Clark,unknown,https://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film),"Alice follows a large white rabbit down a ""Rabbit-hole"". She finds a tiny door. When she finds a bottle labeled ""Drink me"", she does, and shrinks, but not enough to pass through the door. She then eats something labeled ""Eat me"" and grows larger. She finds a fan when enables her to shrink enough to get into the ""Garden"" and try to get a ""Dog"" to play with her. She enters the ""White Rabbit's tiny House,"" but suddenly resumes her normal size. In order to get out, she has to use the ""magic fan.""
+She enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. ""The Duchess's Cheshire Cat"" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's ""Mad Tea-Party."" After a while, she leaves.
+The Queen invites Alice to join the ""ROYAL PROCESSION"": a parade of marching playing cards and others headed by the White Rabbit. When Alice ""unintentionally offends the Queen"", the latter summons the ""Executioner"". Alice ""boxes the ears"", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream."
+1903,The Great Train Robbery,American,Edwin S. Porter,,western,https://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film),"The film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left.
+Meanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water.
+There is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail."
+1904,The Suburbanite,American,Wallace McCutcheon,,comedy,https://en.wikipedia.org/wiki/The_Suburbanite,"The film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest."
+1905,The Little Train Robbery,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Little_Train_Robbery,"The opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The ""Bandit Queen,"" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The ""Bandit Queen"" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin.
+The next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the ""valuables,"" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies.
+In the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the ""plunder."" The police, however, have struck the right trail and are in close pursuit. While the ""plunder"" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the ""Bandit Queen."""
+1905,The Night Before Christmas,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film),"Scenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents."
+1906,Dream of a Rarebit Fiend,American,Wallace McCutcheon and Edwin S. Porter,,short,https://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film),"The Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed."
+1906,From Leadville to Aspen: A Hold-Up in the Rockies,American,Francis J. Marion and Wallace McCutcheon,,short action/crime western,https://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_Rockies,The film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart.
+1906,Kathleen Mavourneen,American,Edwin S. Porter,,short film,https://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film),"Irish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration.
+Film historian Charles Musser writes of Porter's adaptation, ""O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.""[1]"
+1907,Daniel Boone,American,Wallace McCutcheon and Ediwin S. Porter,"William Craven, Florence Lawrence",biographical,https://en.wikipedia.org/wiki/Daniel_Boone_(1907_film),"Boone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]"
+1907,How Brown Saw the Baseball Game,American,Unknown,Unknown,comedy,https://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_Game,"Before heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]"
+1907,Laughing Gas,American,Edwin Stanton Porter,"Bertha Regustus, Edward Boulden",comedy,https://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_Film,"The plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets ""catches"" the laughter from her, including a vendor and police officers."
+1908,The Adventures of Dollie,American,D. W. Griffith,"Arthur V. Johnson, Linda Arvidson",drama,https://en.wikipedia.org/wiki/The_Adventures_of_Dollie,"On a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents."
+1908,The Black Viper,American,D. W. Griffith,D. W. Griffith,drama,https://en.wikipedia.org/wiki/The_Black_Viper,"A thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house."
+1908,A Calamitous Elopement,American,D.W. Griffith,"Harry Solter, Linda Arvidson",comedy,https://en.wikipedia.org/wiki/A_Calamitous_Elopement,"A young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings."
+1908,The Call of the Wild,American,D. W. Griffith,Charles Inslee,adventure,https://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film),"A white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as ""The Biograph Girl."""
+1908,A Christmas Carol,American,Unknown,Tom Ricketts,drama,https://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film),"No prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life."
+1908,The Fight for Freedom,American,D. W. Griffith,"Florence Auer, John G. Adolfi",western,https://en.wikipedia.org/wiki/The_Fight_for_Freedom,"The film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town."

scripts/image/test-all-outbound-connectivity-scenarios.sh

@@ -0,0 +1,61 @@
+#!/usr/bin/env bash
+
+# Note:
+#
+#  The scenarios baseline, missing-models, and analytics-online-only
+#  are expected to have conversations reported by tshark
+#
+#  The scenarios offline and offline-and-missing-models
+#  are *NOT* expected to have any conversations (or attempted conversations) reported by tshark
+
+set -euo pipefail
+
+# shellcheck disable=SC2015
+((BASH_VERSINFO[0] >= 5)) || {
+  echo "Requires bash >= 5" >&2
+  exit 1
+}
+
+mkdir -p python-output
+mkdir -p pcaps
+
+start_timestamp_seconds=$(date +%s)
+
+./test-outbound-connectivity.sh --cleanup baseline
+./test-outbound-connectivity.sh --cleanup missing-models
+./test-outbound-connectivity.sh --cleanup analytics-online-only
+./test-outbound-connectivity.sh --cleanup offline
+./test-outbound-connectivity.sh --cleanup offline-and-missing-models
+
+set +e
+found_pcap_files=$(find "pcaps" -maxdepth 1 -name "*.pcap" -type f -newermt "@$start_timestamp_seconds" 2>/dev/null | wc -l | tr -d ' ')
+found_log_files=$(find "python-output" -maxdepth 1 -name "*.log" -type f -newermt "@$start_timestamp_seconds" 2>/dev/null | wc -l | tr -d ' ')
+set -e
+if [ "$found_pcap_files" -ne "5" ]; then
+  echo "Expected to find 4 fresh pcap/ files from this test but found $found_pcap_files instead"
+  exit 1
+fi
+if [ "$found_log_files" -ne "5" ]; then
+  echo "Expected to find 4 fresh python-output .log files from this test but found $found_log_files instead"
+  exit 1
+fi
+
+for scenario in baseline missing-models analytics-online-only offline offline-and-missing-models; do
+  echo
+
+  echo "=================================================================="
+  echo "======================================== Begin Scenario: $scenario"
+  echo
+  echo "   -------------------------------------------"
+  echo "   tshark output for $scenario"
+  echo "   -------------------------------------------"
+  echo
+  tshark -r pcaps/$scenario.pcap -q -z conv,ip | grep -v '===================================='
+
+  echo
+  echo "   ------------------------------------------"
+  echo "   python log output for $scenario"
+  echo "   ------------------------------------------"
+  echo
+  cat python-output/$scenario.log
+done

scripts/image/test-outbound-connectivity.sh

@@ -0,0 +1,188 @@
+#!/usr/bin/env bash
+#
+# test-outbound-connectivity.sh
+#
+# Capture every external packet an Unstructured Docker image emits while
+# partition()‑ing a test PNG, *inside the same container* (works on macOS).
+#
+# In addition **also capture the Python workload's stdout / stderr** and save it
+# under ./python-output/<scenario>.log while still streaming it to your terminal.
+#
+# Usage examples
+#   ./test-outbound-connectivity.sh baseline
+#   ./test-outbound-connectivity.sh --cleanup missing-models
+#   ./test-outbound-connectivity.sh --cleanup offline
+#   ./test-outbound-connectivity.sh offline-and-missing-models
+#
+# Outputs:
+#   ./pcaps/<scenario>.pcap
+#   ./python-output/<scenario>.log
+# ---------------------------------------------------------------------
+
+set -euo pipefail
+
+######################## user‑tunable constants ########################
+IMAGE="downloads.unstructured.io/unstructured-io/unstructured:e42884a"
+NET="unstructured_test_net"
+CAPTURE_IFACE="${CAPTURE_IFACE:-eth0}"
+PCAP_DIR="$(pwd)/pcaps"
+PY_LOG_DIR="$(pwd)/python-output" # where Python logs go
+HF_CACHE="/home/notebook-user/.cache/huggingface"
+########################################################################
+
+# shellcheck disable=SC2015
+((BASH_VERSINFO[0] >= 5)) || {
+  echo "Requires bash >= 5" >&2
+  exit 1
+}
+
+# Create output directories up‑front so failures don’t leave us empty‑handed
+mkdir -p "$PCAP_DIR" "$PY_LOG_DIR"
+
+# ---------- parse flags (optional --cleanup) --------------------------
+CLEANUP=0
+if [[ "${1:-}" == "--cleanup" ]]; then
+  CLEANUP=1
+  shift
+fi
+
+SCENARIO="${1:-}"
+if [[ -z "$SCENARIO" ]]; then
+  echo "Usage: $0 [--cleanup] {baseline|missing-models|analytics-online-only|offline|offline-and-missing-models}" >&2
+  exit 1
+fi
+
+# ---------- optional pre‑run cleanup ----------------------------------
+if ((CLEANUP)); then
+  echo ">>> Removing leftover sut_* containers…"
+  # shellcheck disable=SC2015
+  docker rm -f "$(docker ps -aq --filter name='^sut_')" 2>/dev/null || true
+fi
+
+# ---------- scenario‑specific settings --------------------------------
+DO_NOT_TRACK=""
+UNSTRUCTURED_TELEMETRY_ENABLED=""
+HF_HUB_OFFLINE=""
+REMOVE_CACHE=0
+case "$SCENARIO" in
+baseline) ;;
+missing-models) REMOVE_CACHE=1 ;;
+analytics-online-only)
+  UNSTRUCTURED_TELEMETRY_ENABLED=1
+  HF_HUB_OFFLINE=1
+  ;;
+offline)
+  DO_NOT_TRACK=true
+  HF_HUB_OFFLINE=1
+  ;;
+offline-and-missing-models)
+  DO_NOT_TRACK=true
+  HF_HUB_OFFLINE=1
+  REMOVE_CACHE=1
+  ;;
+*)
+  echo "Unknown scenario: $SCENARIO"
+  exit 1
+  ;;
+esac
+
+docker network inspect "$NET" >/dev/null 2>&1 || docker network create "$NET"
+
+# ---------- launch SUT idle -------------------------------------------
+CID=$(docker run -d --rm --name "sut_${SCENARIO}" \
+  --network "$NET" \
+  --cap-add NET_RAW --cap-add NET_ADMIN \
+  -e DO_NOT_TRACK="$DO_NOT_TRACK" \
+  -e UNSTRUCTURED_TELEMETRY_ENABLED="$UNSTRUCTURED_TELEMETRY_ENABLED" \
+  -e HF_HUB_OFFLINE="$HF_HUB_OFFLINE" \
+  --entrypoint /bin/sh "$IMAGE" -c "sleep infinity")
+echo "Container: $CID  (scenario $SCENARIO)"
+
+# install tcpdump (Wolfi uses apk) as root
+docker exec -u root "$CID" apk add --no-cache tcpdump >/dev/null
+
+# optionally wipe HF cache
+# shellcheck disable=SC2015
+((REMOVE_CACHE)) && docker exec "$CID" rm -rf "$HF_CACHE" || true
+
+# ---------- start tcpdump in background -------------------------------
+FILTER='not (dst net ff02::/16 or src net ff02::/16 or ip6[6] = 58 or ether multicast)'
+
+docker exec -u root -d "$CID" sh -c "tcpdump -U -n -i $CAPTURE_IFACE '$FILTER' -w /tmp/capture.pcap > /tmp/tcpdump.log 2>&1"
+
+# check if tcpdump stayed alive
+sleep 2
+if ! docker exec "$CID" pgrep tcpdump >/dev/null; then
+  echo 'tcpdump exited – showing its log:'
+  docker exec "$CID" cat /tmp/tcpdump.log
+  exit 1
+fi
+
+echo "tcpdump running on interface $CAPTURE_IFACE..."
+# ---------- run the Python workload -----------------------------------
+echo ">>> Running Python workload (capturing stdout/stderr)…"
+#   ‑ The "|&" pipes *both* stdout *and* stderr into tee.
+#   ‑ tee sends it to the terminal *and* writes the log file.
+#   ‑ With `set -o pipefail` we still fail early if the Python process exits non‑zero.
+
+if [[ "$HF_HUB_OFFLINE" -eq 1 && "$REMOVE_CACHE" -eq 1 ]]; then
+  echo "HF_HUB_OFFLINE=1 and REMOVE_CACHE=1 : allowing python command have a non-exit 0 status and will continue the script."
+  set +e
+fi
+
+docker exec -i -e PYTHONUNBUFFERED=1 "$CID" python - <<PY |& tee "${PY_LOG_DIR}/${SCENARIO}.log"
+import logging
+# Telemetry runs at package init when UNSTRUCTURED_TELEMETRY_ENABLED is set (see analytics-online-only scenario).
+from unstructured.partition.auto import partition
+import urllib.request, time, os, sys
+
+# Configure detailed logging
+logging.basicConfig(
+    level=logging.INFO,
+    format='%(asctime)s - %(name)s - %(levelname)s - %(message)s',
+    handlers=[
+        logging.StreamHandler(sys.stdout),
+        logging.FileHandler('test_platform_api.log')
+    ]
+)
+logging.getLogger("urllib").setLevel(logging.DEBUG)
+logging.getLogger("urllib3").setLevel(logging.DEBUG)
+logging.getLogger("httpx").setLevel(logging.DEBUG)
+logging.getLogger("httpcore").setLevel(logging.DEBUG)
+logging.getLogger("pdfminer.pdfpage").setLevel(logging.CRITICAL)
+
+for test_file in [
+ "/app/example-docs/ideas-page.html",
+ "/app/example-docs/category-level.docx",
+ "/app/example-docs/fake_table.docx",
+ "/app/example-docs/img/english-and-korean.png",
+ "/app/example-docs/img/embedded-images-tables.jpg",
+ "/app/example-docs/img/layout-parser-paper-with-table.jpg",
+ "/app/example-docs/pdf/embedded-images-tables.pdf",
+ "/app/example-docs/pdf/all-number-table.pdf",
+ "/app/example-docs/fake-power-point.pptx",
+ "/app/example-docs/stanley-cups.xlsx",
+ "/app/example-docs/fake-email-multiple-attachments.msg",
+]:
+ print("[INFO] partitioning "+test_file)
+ partition(test_file, strategy="hi_res", skip_infer_table_types=[])
+## add this if you always want to force an external connection
+#print("[INFO] done partitioning; hitting google…")
+#urllib.request.urlopen("https://www.google.com", timeout=10).read(64)
+#print("[INFO] google fetch finished")
+
+time.sleep(1)  # ensure FIN packets captured
+PY
+
+echo "Python finished.  Log saved to ${PY_LOG_DIR}/${SCENARIO}.log"
+
+set -e
+
+# ---------- stop tcpdump, copy pcap, clean up -------------------------
+docker exec "$CID" pkill -2 tcpdump || true
+sleep 1 # let pcap flush
+
+docker cp "$CID:/tmp/capture.pcap" "${PCAP_DIR}/${SCENARIO}.pcap"
+echo "pcap saved to ${PCAP_DIR}/${SCENARIO}.pcap"
+
+docker stop "$CID" >/dev/null

scripts/ingest-test-fixtures-update.sh

@@ -0,0 +1,58 @@
+#!/usr/bin/env bash
+
+# Structured .json output from PDF's or images may differ subtly (or not so subtly)
+# based on the version of tesseract, its dependencies, and chip architecture.
+#
+# To update ingest-test expected outputs (structured .json files), this script:
+#   * builds an ubuntu image that
+#      * matches CI with respect to tesseract and OS deps
+#      * installs python dependencies via uv
+#   * runs each test ingest script with OVERWRITE_FIXTURES=true
+#      * so updates are written to test_unstructured_ingest/expected-structured-output/
+#      * using local unstructured/ directory (i.e. from local git branch)
+#
+# It is recommended to run this script on x86_64 hardware.
+
+set -eu -o pipefail
+
+# Change to the root of the repository
+SCRIPT_DIR=$(cd -- "$(dirname -- "${BASH_SOURCE[0]}")" &>/dev/null && pwd)
+cd "$SCRIPT_DIR"/.. || exit 1
+
+ARCHITECTURE=$(uname -m)
+
+if [ "$ARCHITECTURE" != "x86_64" ]; then
+  echo "Warning: This script is designed to run on x86_64 hardware, but you're running on $ARCHITECTURE."
+fi
+
+./scripts/docker-build-ubuntu.sh
+
+# Warn the user if they have an old image
+IMAGE_NAME="unstructured-ubuntu:latest"
+CREATION_TIMESTAMP=$(docker inspect --format='{{.Created}}' "$IMAGE_NAME")
+CREATION_DATE=$(date -d "$CREATION_TIMESTAMP" +%s)
+CURRENT_DATE=$(date +%s)
+AGE_DAYS=$(((CURRENT_DATE - CREATION_DATE) / 86400))
+if [ "$AGE_DAYS" -gt 6 ]; then
+  echo "WARNING: The image \"$IMAGE_NAME\" is more than 7 days old ($AGE_DAYS days)."
+  echo "You may want to 'docker rmi $IMAGE_NAME' and rerun this script if it is not current."
+fi
+
+docker run --rm -v "$SCRIPT_DIR"/../unstructured:/root/unstructured \
+  -v "$SCRIPT_DIR"/../test_unstructured_ingest:/root/test_unstructured_ingest \
+  ${DISCORD_TOKEN:+-e DISCORD_TOKEN="$DISCORD_TOKEN"} \
+  ${SLACK_TOKEN:+-e SLACK_TOKEN="$SLACK_TOKEN"} \
+  ${CONFLUENCE_USER_EMAIL:+-e CONFLUENCE_USER_EMAIL="$CONFLUENCE_USER_EMAIL"} \
+  ${CONFLUENCE_API_TOKEN:+-e CONFLUENCE_API_TOKEN="$CONFLUENCE_API_TOKEN"} \
+  ${GH_READ_ONLY_ACCESS_TOKEN:+-e GH_READ_ONLY_ACCESS_TOKEN="$GH_READ_ONLY_ACCESS_TOKEN"} \
+  -w /root "$IMAGE_NAME" \
+  bash -c "export OVERWRITE_FIXTURES=true && source ~/.bashrc && pyenv activate unstructured && tesseract --version &&
+               ./test_unstructured_ingest/test-ingest-azure.sh &&
+               ./test_unstructured_ingest/test-ingest-discord.sh &&
+               ./test_unstructured_ingest/test-ingest-github.sh &&
+               ./test_unstructured_ingest/test-ingest-biomed-api.sh &&
+               ./test_unstructured_ingest/test-ingest-biomed-path.sh &&
+               ./test_unstructured_ingest/test-ingest-s3.sh &&
+               ./test_unstructured_ingest/test-ingest-slack.sh &&
+               ./test_unstructured_ingest/test-ingest-pdf-fast-reprocess.sh &&
+               ./test_unstructured_ingest/test-ingest-local-single-file-with-pdf-infer-table-structure.sh"

scripts/initialize-libreoffice.sh

@@ -0,0 +1,3 @@
+#!/bin/bash
+
+/usr/bin/soffice --headless || [ $? -eq 81 ] || exit 1

scripts/kafka-test-helpers/create-kafka-instance.sh

@@ -0,0 +1,12 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+
+# Create the Weaviate instance
+docker compose version
+docker compose -f "$SCRIPT_DIR"/docker-compose.yml up --wait
+docker compose -f "$SCRIPT_DIR"/docker-compose.yml ps
+
+echo "Instance is live."

scripts/kafka-test-helpers/docker-compose.yml

@@ -0,0 +1,22 @@
+services:
+  zookeeper:
+    image: confluentinc/cp-zookeeper:latest
+    environment:
+      ZOOKEEPER_CLIENT_PORT: 2181
+      ZOOKEEPER_TICK_TIME: 2000
+    ports:
+      - 22181:2181
+
+  kafka:
+    image: confluentinc/cp-kafka:latest
+    depends_on:
+      - zookeeper
+    ports:
+      - 29092:29092
+    environment:
+      KAFKA_BROKER_ID: 1
+      KAFKA_ZOOKEEPER_CONNECT: zookeeper:2181
+      KAFKA_ADVERTISED_LISTENERS: PLAINTEXT://kafka:9092,PLAINTEXT_HOST://localhost:29092
+      KAFKA_LISTENER_SECURITY_PROTOCOL_MAP: PLAINTEXT:PLAINTEXT,PLAINTEXT_HOST:PLAINTEXT
+      KAFKA_INTER_BROKER_LISTENER_NAME: PLAINTEXT
+      KAFKA_OFFSETS_TOPIC_REPLICATION_FACTOR: 1

scripts/minio-test-helpers/create-and-check-minio.sh

@@ -0,0 +1,25 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+
+secret_key=minioadmin
+access_key=minioadmin
+region=us-east-2
+endpoint_url=http://localhost:9000
+bucket_name=utic-dev-tech-fixtures
+
+function upload() {
+  echo "Uploading test content to new bucket in minio"
+  AWS_REGION=$region AWS_SECRET_ACCESS_KEY=$secret_key AWS_ACCESS_KEY_ID=$access_key \
+    aws --output json --endpoint-url $endpoint_url s3api create-bucket --bucket $bucket_name | jq
+  AWS_REGION=$region AWS_SECRET_ACCESS_KEY=$secret_key AWS_ACCESS_KEY_ID=$access_key \
+    aws --endpoint-url $endpoint_url s3 cp "$SCRIPT_DIR"/wiki_movie_plots_small.csv s3://$bucket_name/
+}
+
+# Create Minio single server
+docker compose version
+docker compose -f "$SCRIPT_DIR"/docker-compose.yaml up --wait
+docker compose -f "$SCRIPT_DIR"/docker-compose.yaml ps
+
+echo "Cluster is live."
+upload

scripts/minio-test-helpers/docker-compose.yaml

@@ -0,0 +1,13 @@
+services:
+  minio:
+    image: quay.io/minio/minio
+    container_name: minio-test
+    ports:
+      - 9000:9000
+      - 9001:9001
+    command: server --console-address ":9001" /data
+    healthcheck:
+      test: ["CMD", "curl", "-f", "http://localhost:9000/minio/health/live"]
+      interval: 5s
+      timeout: 20s
+      retries: 3

scripts/minio-test-helpers/wiki_movie_plots_small.csv

@@ -0,0 +1,31 @@
+Release Year,Title,Origin/Ethnicity,Director,Cast,Genre,Wiki Page,Plot
+1901,Kansas Saloon Smashers,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Kansas_Saloon_Smashers,"A bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]"
+1901,Love by the Light of the Moon,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Love_by_the_Light_of_the_Moon,"The moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better."
+1901,The Martyred Presidents,American,Unknown,,unknown,https://en.wikipedia.org/wiki/The_Martyred_Presidents,"The film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination.
+In the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice."
+1901,"Terrible Teddy, the Grizzly King",American,Unknown,,unknown,"https://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_King","Lasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading ""His Photographer"" and ""His Press Agent"" respectively, follow him into the shot; the photographer sets up his camera. ""Teddy"" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. ""Teddy"" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. ""Teddy"" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs."
+1902,Jack and the Beanstalk,American,"George S. Fleming, Edwin S. Porter",,unknown,https://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film),"The earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince."
+1903,Alice in Wonderland,American,Cecil Hepworth,May Clark,unknown,https://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film),"Alice follows a large white rabbit down a ""Rabbit-hole"". She finds a tiny door. When she finds a bottle labeled ""Drink me"", she does, and shrinks, but not enough to pass through the door. She then eats something labeled ""Eat me"" and grows larger. She finds a fan when enables her to shrink enough to get into the ""Garden"" and try to get a ""Dog"" to play with her. She enters the ""White Rabbit's tiny House,"" but suddenly resumes her normal size. In order to get out, she has to use the ""magic fan.""
+She enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. ""The Duchess's Cheshire Cat"" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's ""Mad Tea-Party."" After a while, she leaves.
+The Queen invites Alice to join the ""ROYAL PROCESSION"": a parade of marching playing cards and others headed by the White Rabbit. When Alice ""unintentionally offends the Queen"", the latter summons the ""Executioner"". Alice ""boxes the ears"", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream."
+1903,The Great Train Robbery,American,Edwin S. Porter,,western,https://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film),"The film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left.
+Meanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water.
+There is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail."
+1904,The Suburbanite,American,Wallace McCutcheon,,comedy,https://en.wikipedia.org/wiki/The_Suburbanite,"The film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest."
+1905,The Little Train Robbery,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Little_Train_Robbery,"The opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The ""Bandit Queen,"" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The ""Bandit Queen"" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin.
+The next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the ""valuables,"" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies.
+In the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the ""plunder."" The police, however, have struck the right trail and are in close pursuit. While the ""plunder"" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the ""Bandit Queen."""
+1905,The Night Before Christmas,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film),"Scenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents."
+1906,Dream of a Rarebit Fiend,American,Wallace McCutcheon and Edwin S. Porter,,short,https://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film),"The Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed."
+1906,From Leadville to Aspen: A Hold-Up in the Rockies,American,Francis J. Marion and Wallace McCutcheon,,short action/crime western,https://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_Rockies,The film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart.
+1906,Kathleen Mavourneen,American,Edwin S. Porter,,short film,https://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film),"Irish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration.
+Film historian Charles Musser writes of Porter's adaptation, ""O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.""[1]"
+1907,Daniel Boone,American,Wallace McCutcheon and Ediwin S. Porter,"William Craven, Florence Lawrence",biographical,https://en.wikipedia.org/wiki/Daniel_Boone_(1907_film),"Boone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]"
+1907,How Brown Saw the Baseball Game,American,Unknown,Unknown,comedy,https://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_Game,"Before heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]"
+1907,Laughing Gas,American,Edwin Stanton Porter,"Bertha Regustus, Edward Boulden",comedy,https://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_Film,"The plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets ""catches"" the laughter from her, including a vendor and police officers."
+1908,The Adventures of Dollie,American,D. W. Griffith,"Arthur V. Johnson, Linda Arvidson",drama,https://en.wikipedia.org/wiki/The_Adventures_of_Dollie,"On a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents."
+1908,The Black Viper,American,D. W. Griffith,D. W. Griffith,drama,https://en.wikipedia.org/wiki/The_Black_Viper,"A thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house."
+1908,A Calamitous Elopement,American,D.W. Griffith,"Harry Solter, Linda Arvidson",comedy,https://en.wikipedia.org/wiki/A_Calamitous_Elopement,"A young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings."
+1908,The Call of the Wild,American,D. W. Griffith,Charles Inslee,adventure,https://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film),"A white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as ""The Biograph Girl."""
+1908,A Christmas Carol,American,Unknown,Tom Ricketts,drama,https://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film),"No prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life."
+1908,The Fight for Freedom,American,D. W. Griffith,"Florence Auer, John G. Adolfi",western,https://en.wikipedia.org/wiki/The_Fight_for_Freedom,"The film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town."

scripts/opensearch-test-helpers/common/docker-compose.yaml

@@ -0,0 +1,14 @@
+services:
+  opensearch:
+    image: opensearchproject/opensearch:2.19.4
+    container_name: opensearch-test
+    ports:
+      - 9247:9200
+      - 9600:9600
+    environment:
+      - discovery.type=single-node
+    healthcheck:
+      test: curl --fail https://localhost:9200/_cat/health -ku 'admin:admin' >/dev/null || exit 1
+      interval: 10s
+      timeout: 30s
+      retries: 3

scripts/opensearch-test-helpers/destination_connector/create-opensearch-instance.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(dirname "$(dirname "$(realpath "$0")")")
+
+# Create the Opensearch cluster
+docker compose version
+docker compose -f "$SCRIPT_DIR"/common/docker-compose.yaml up --wait
+docker compose -f "$SCRIPT_DIR"/common/docker-compose.yaml ps
+
+echo "Cluster is live."
+python "$SCRIPT_DIR"/destination_connector/create_index.py

scripts/opensearch-test-helpers/destination_connector/create_index.py

@@ -0,0 +1,24 @@
+#!/usr/bin/env python3
+
+from opensearch_cluster_config import (
+    INDEX_NAME,
+    mappings,
+)
+from opensearchpy import OpenSearch
+
+print("Connecting to the OpenSearch cluster.")
+client = OpenSearch(
+    hosts=[{"host": "localhost", "port": 9247}],
+    http_auth=("admin", "admin"),
+    use_ssl=True,
+    verify_certs=False,
+    ssl_show_warn=False,
+)
+print(client.info())
+
+print("Creating an OpenSearch index for testing ingest opensearch destination connector.")
+response = client.indices.create(index=INDEX_NAME, body=mappings)
+if not response["acknowledged"]:
+    raise RuntimeError("failed to create index")
+
+print("Succesfully created an OpenSearch index for testing opensearch ingest.")

scripts/opensearch-test-helpers/destination_connector/opensearch_cluster_config.py

@@ -0,0 +1,12 @@
+import json
+
+CLUSTER_URL = "http://localhost:9247"
+INDEX_NAME = "ingest-test-destination"
+USER = "admin"
+PASSWORD = "admin"
+MAPPING_PATH = (
+    "scripts/opensearch-test-helpers/destination_connector/opensearch_elements_mappings.json"
+)
+
+with open(MAPPING_PATH) as f:
+    mappings = json.load(f)

scripts/opensearch-test-helpers/destination_connector/opensearch_elements_mappings.json

@@ -0,0 +1,149 @@
+{"settings": {
+   "index": {
+     "knn": true,
+     "knn.algo_param.ef_search": 100
+   }
+ },
+   "mappings": {
+      "properties": {
+         "element_id": {
+            "type": "keyword"
+         },
+         "text": {
+            "type": "text",
+            "analyzer": "english"
+         },
+         "type": {
+            "type": "text"
+         },
+         "embeddings": {
+            "type": "knn_vector",
+            "dimension": 384
+         },
+         "metadata": {
+            "type": "object",
+            "properties": {
+               "category_depth": {
+                  "type": "integer"
+               },
+               "parent_id": {
+                  "type": "keyword"
+               },
+               "attached_to_filename": {
+                  "type": "keyword"
+               },
+               "filetype": {
+                  "type": "keyword"
+               },
+               "last_modified": {
+                  "type": "date"
+               },
+               "file_directory": {
+                  "type": "keyword"
+               },
+               "filename": {
+                  "type": "keyword"
+               },
+               "data_source": {
+                  "type": "object",
+                  "properties": {
+                     "url": {
+                        "type": "text",
+                        "analyzer": "standard"
+                     },
+                     "version": {
+                        "type": "keyword"
+                     },
+                     "date_created": {
+                        "type": "date"
+                     },
+                     "date_modified": {
+                        "type": "date"
+                     },
+                     "date_processed": {
+                        "type": "date"
+                     },
+                     "record_locator": {
+                        "type": "keyword"
+                     },
+                     "permissions_data": {
+                        "type": "object"
+                     }
+                  }
+               },
+               "coordinates": {
+                  "type": "object",
+                  "properties": {
+                     "system": {
+                        "type": "keyword"
+                     },
+                     "layout_width": {
+                        "type": "float"
+                     },
+                     "layout_height": {
+                        "type": "float"
+                     },
+                     "points": {
+                        "type": "float"
+                     }
+                  }
+               },
+               "languages": {
+                  "type": "keyword"
+               },
+               "page_number": {
+                  "type": "integer"
+               },
+               "page_name": {
+                  "type": "keyword"
+               },
+               "url": {
+                  "type": "text",
+                  "analyzer": "standard"
+               },
+               "links": {
+                  "type": "object"
+               },
+               "link_urls": {
+                  "type": "text"
+               },
+               "link_texts": {
+                  "type": "text"
+               },
+               "sent_from": {
+                  "type": "text",
+                  "analyzer": "standard"
+               },
+               "sent_to": {
+                  "type": "text",
+                  "analyzer": "standard"
+               },
+               "subject": {
+                  "type": "text",
+                  "analyzer": "standard"
+               },
+               "section": {
+                  "type": "text",
+                  "analyzer": "standard"
+               },
+               "header_footer_type": {
+                  "type": "keyword"
+               },
+               "emphasized_text_contents": {
+                  "type": "text"
+               },
+               "emphasized_text_tags": {
+                  "type": "keyword"
+               },
+               "text_as_html": {
+                  "type": "text",
+                  "analyzer": "standard"
+               },
+               "detection_class_prob": {
+                  "type": "float"
+               }
+            }
+         }
+      }
+   }
+}

scripts/opensearch-test-helpers/destination_connector/test-ingest-opensearch-output.py

@@ -0,0 +1,59 @@
+#!/usr/bin/env python3
+import time
+
+from opensearchpy import OpenSearch
+
+N_ELEMENTS = 5
+EXPECTED_TEXT = "To Whom it May Concern:"
+
+if __name__ == "__main__":
+    print("Connecting to the OpenSearch cluster.")
+    client = OpenSearch(
+        hosts=[{"host": "localhost", "port": 9247}],
+        http_auth=("admin", "admin"),
+        use_ssl=True,
+        verify_certs=False,
+        ssl_show_warn=False,
+    )
+    print(client.info())
+
+    initial_query = {"query": {"simple_query_string": {"fields": ["text"], "query": EXPECTED_TEXT}}}
+
+    initial_embeddings = None
+    timeout_s = 9
+    sleep_s = 1
+
+    start = time.time()
+    found = False
+    while time.time() - start < timeout_s and not found:
+        results = client.search(index="ingest-test-destination", body=initial_query)
+        hits = results["hits"]["hits"]
+        if hits:
+            print(f"found results after {time.time() - start}s")
+            initial_embeddings = hits[0]["_source"]["embeddings"]
+            found = True
+            break
+        print(f"Waiting {sleep_s}s before checking again")
+        time.sleep(sleep_s)
+
+    if not found:
+        raise TimeoutError(
+            f"timed out after {round(timeout_s, 3)}s trying to get results from opensearch"
+        )
+
+    query = {"size": 1, "query": {"knn": {"embeddings": {"vector": initial_embeddings, "k": 1}}}}
+
+    vector_search = client.search(index="ingest-test-destination", body=query)
+
+    found_text = vector_search["hits"]["hits"][0]["_source"]["text"]
+    assert found_text == EXPECTED_TEXT, (
+        f"OpenSearch dest check failed: Did not find "
+        f"{EXPECTED_TEXT} in via vector search, instead: {found_text}."
+    )
+    print("OpenSearch vector search test was successful.")
+
+    count = client.count(index="ingest-test-destination")["count"]
+
+    assert int(count) == N_ELEMENTS, f"OpenSearch dst check fail: expect {N_ELEMENTS} got {count}"
+
+    print(f"OpenSearch destination test was successful with {count} items being uploaded.")

scripts/opensearch-test-helpers/source_connector/create-and-check-opensearch.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR="$(dirname "$(dirname "$(realpath "$0")")")"
+
+# Create the Opensearch cluster
+docker compose version
+docker compose -f "$SCRIPT_DIR"/common/docker-compose.yaml up --wait
+docker compose -f "$SCRIPT_DIR"/common/docker-compose.yaml ps
+
+echo "Cluster is live."
+"$SCRIPT_DIR"/source_connector/create_and_fill_opensearch.py

scripts/opensearch-test-helpers/source_connector/create_and_fill_opensearch.py

@@ -0,0 +1,66 @@
+#!/usr/bin/env python3
+from contextlib import suppress
+
+import pandas as pd
+from opensearchpy import Document, Keyword, OpenSearch, Text
+from opensearchpy.exceptions import NotFoundError
+
+DATA_PATH = "scripts/opensearch-test-helpers/wiki_movie_plots_small.csv"
+CLUSTER_URL = "http://localhost:9247"
+INDEX_NAME = "movies"
+
+
+class Movie(Document):
+    title = Text(fields={"raw": Keyword()})
+    year = Text()
+    director = Text()
+    cast = Text()
+    genre = Text()
+    wiki_page = Text()
+    ethnicity = Text()
+    plot = Text()
+
+    class Index:
+        name = "movies"
+
+    def save(self, **kwargs):
+        return super(Movie, self).save(**kwargs)
+
+
+print("Connecting to the OpenSearch cluster.")
+client = OpenSearch(
+    hosts=[{"host": "localhost", "port": 9247}],
+    http_auth=("admin", "admin"),
+    use_ssl=True,
+    verify_certs=False,
+    ssl_show_warn=False,
+)
+print(client.info())
+df = pd.read_csv(DATA_PATH).dropna().reset_index()
+
+with suppress(NotFoundError):
+    client.indices.delete(index="movies")
+
+print("Creating an OpenSearch index for testing opensearch ingest.")
+response = client.indices.create(index=INDEX_NAME)
+if not response.get("acknowledged"):
+    raise RuntimeError("failed to create index")
+
+for i, row in df.iterrows():
+    Movie.init(using=client)
+    movie = Movie(
+        meta={"id": i},
+        title=row["Title"],
+        year=row["Release Year"],
+        director=row["Director"],
+        cast=row["Cast"],
+        genre=row["Genre"],
+        wiki_page=row["Wiki Page"],
+        ethnicity=row["Origin/Ethnicity"],
+        plot=row["Plot"],
+    )
+    movie.save(using=client)
+
+client.count()
+
+print("Successfully created and filled an OpenSearch index for testing opensearch ingest.")

scripts/opensearch-test-helpers/wiki_movie_plots_small.csv

@@ -0,0 +1,31 @@
+Release Year,Title,Origin/Ethnicity,Director,Cast,Genre,Wiki Page,Plot
+1901,Kansas Saloon Smashers,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Kansas_Saloon_Smashers,"A bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]"
+1901,Love by the Light of the Moon,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Love_by_the_Light_of_the_Moon,"The moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better."
+1901,The Martyred Presidents,American,Unknown,,unknown,https://en.wikipedia.org/wiki/The_Martyred_Presidents,"The film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination.
+In the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice."
+1901,"Terrible Teddy, the Grizzly King",American,Unknown,,unknown,"https://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_King","Lasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading ""His Photographer"" and ""His Press Agent"" respectively, follow him into the shot; the photographer sets up his camera. ""Teddy"" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. ""Teddy"" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. ""Teddy"" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs."
+1902,Jack and the Beanstalk,American,"George S. Fleming, Edwin S. Porter",,unknown,https://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film),"The earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince."
+1903,Alice in Wonderland,American,Cecil Hepworth,May Clark,unknown,https://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film),"Alice follows a large white rabbit down a ""Rabbit-hole"". She finds a tiny door. When she finds a bottle labeled ""Drink me"", she does, and shrinks, but not enough to pass through the door. She then eats something labeled ""Eat me"" and grows larger. She finds a fan when enables her to shrink enough to get into the ""Garden"" and try to get a ""Dog"" to play with her. She enters the ""White Rabbit's tiny House,"" but suddenly resumes her normal size. In order to get out, she has to use the ""magic fan.""
+She enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. ""The Duchess's Cheshire Cat"" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's ""Mad Tea-Party."" After a while, she leaves.
+The Queen invites Alice to join the ""ROYAL PROCESSION"": a parade of marching playing cards and others headed by the White Rabbit. When Alice ""unintentionally offends the Queen"", the latter summons the ""Executioner"". Alice ""boxes the ears"", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream."
+1903,The Great Train Robbery,American,Edwin S. Porter,,western,https://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film),"The film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left.
+Meanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water.
+There is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail."
+1904,The Suburbanite,American,Wallace McCutcheon,,comedy,https://en.wikipedia.org/wiki/The_Suburbanite,"The film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest."
+1905,The Little Train Robbery,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Little_Train_Robbery,"The opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The ""Bandit Queen,"" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The ""Bandit Queen"" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin.
+The next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the ""valuables,"" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies.
+In the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the ""plunder."" The police, however, have struck the right trail and are in close pursuit. While the ""plunder"" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the ""Bandit Queen."""
+1905,The Night Before Christmas,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film),"Scenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents."
+1906,Dream of a Rarebit Fiend,American,Wallace McCutcheon and Edwin S. Porter,,short,https://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film),"The Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed."
+1906,From Leadville to Aspen: A Hold-Up in the Rockies,American,Francis J. Marion and Wallace McCutcheon,,short action/crime western,https://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_Rockies,The film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart.
+1906,Kathleen Mavourneen,American,Edwin S. Porter,,short film,https://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film),"Irish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration.
+Film historian Charles Musser writes of Porter's adaptation, ""O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.""[1]"
+1907,Daniel Boone,American,Wallace McCutcheon and Ediwin S. Porter,"William Craven, Florence Lawrence",biographical,https://en.wikipedia.org/wiki/Daniel_Boone_(1907_film),"Boone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]"
+1907,How Brown Saw the Baseball Game,American,Unknown,Unknown,comedy,https://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_Game,"Before heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]"
+1907,Laughing Gas,American,Edwin Stanton Porter,"Bertha Regustus, Edward Boulden",comedy,https://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_Film,"The plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets ""catches"" the laughter from her, including a vendor and police officers."
+1908,The Adventures of Dollie,American,D. W. Griffith,"Arthur V. Johnson, Linda Arvidson",drama,https://en.wikipedia.org/wiki/The_Adventures_of_Dollie,"On a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents."
+1908,The Black Viper,American,D. W. Griffith,D. W. Griffith,drama,https://en.wikipedia.org/wiki/The_Black_Viper,"A thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house."
+1908,A Calamitous Elopement,American,D.W. Griffith,"Harry Solter, Linda Arvidson",comedy,https://en.wikipedia.org/wiki/A_Calamitous_Elopement,"A young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings."
+1908,The Call of the Wild,American,D. W. Griffith,Charles Inslee,adventure,https://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film),"A white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as ""The Biograph Girl."""
+1908,A Christmas Carol,American,Unknown,Tom Ricketts,drama,https://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film),"No prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life."
+1908,The Fight for Freedom,American,D. W. Griffith,"Florence Auer, John G. Adolfi",western,https://en.wikipedia.org/wiki/The_Fight_for_Freedom,"The film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town."

scripts/performance/.gitignore

@@ -0,0 +1,2 @@
+benchmark_results
+profile_results

scripts/performance/README.md

@@ -0,0 +1,45 @@
+# Performance
+This is a collection of tools helpful for inspecting and tracking performance of the Unstructured library.
+
+The benchmarking script allows a user to track performance time to partitioning results against a fixed set of test documents and store those results with indication of architecture, instance type, and git hash, in S3.
+
+The profiling script allows a user to inspect how time time and memory are spent across called functions when performing partitioning on a given document.
+
+## Install
+Benchmarking requires no additional dependencies and should work without any initial setup.
+Profiling has a few dependencies which can be installed with:
+
+```bash
+pip install -r scripts/performance/requirements.txt
+npm install -g speedscope
+```
+
+The second dependency `speedscope` provides a tool to view profiling results from `py-spy` locally. Alternatively you can also drop the profile result `*.speedscope` into https://www.speedscope.app/ to view the results online.
+
+## Run
+### Benchmark
+Export / assign desired environment variable settings:
+- DOCKER_TEST: Set to true to run benchmark inside a Docker container (default: false)
+- NUM_ITERATIONS: Number of iterations for benchmark (e.g., 100) (default: 3)
+- INSTANCE_TYPE: Type of benchmark instance (e.g., "c5.xlarge") (default: unspecified)
+- PUBLISH_RESULTS: Set to true to publish results to S3 bucket (default: false)
+-
+Usage: `./scripts/performance/benchmark.sh`
+
+### Profile
+
+Export / assign desired environment variable settings:
+- DOCKER_TEST: Set to true to run profiling inside a Docker container (default: false)
+
+Usage:
+
+**on Linux**: `./scripts/performance/profile.sh`
+
+**on macOS**: `sudo -E ./scripts/performance/profile.sh`; `py-spy` requires su to run on macOS
+
+- Run the script and choose the profiling mode: 'run' or 'view'.
+- In the 'run' mode, you can profile custom files or select existing test files.
+- In the 'view' mode, you can view previously generated profiling results.
+- The script supports time profiling with cProfile and memory profiling with memray.
+- Users can choose different visualization options such as flamegraphs, tables, trees, summaries, and statistics.
+- Test documents are synced from an S3 bucket to a local directory before running the profiles

scripts/performance/benchmark-local.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# This is intended solely to be called by scripts/performance/benchmark.sh.
+# This file is separated out to allow us to easily execute this part of the test script inside a Docker container.
+
+SCRIPT_DIR=$(dirname "$0")
+TEST_DOCS_FOLDER="$SCRIPT_DIR/docs"
+TIMEFORMAT="%R"
+
+mkdir -p "$SCRIPT_DIR/benchmark_results" >/dev/null 2>&1
+DATE=$(date +"%Y-%m-%d_%H-%M-%S")
+RESULTS_FILE="$SCRIPT_DIR/benchmark_results/${DATE}_benchmark_results_${INSTANCE_TYPE}_$("$SCRIPT_DIR/get-stats-name.sh")_$GIT_HASH.csv"
+echo "Test File,Iterations,Average Execution Time (s)" >"$RESULTS_FILE"
+
+echo "Starting benchmark test..."
+for file in "$TEST_DOCS_FOLDER"/*; do
+  echo "Testing file: $(basename "$file")"
+
+  if [[ " ${SLOW_FILES[*]} " =~ $(basename "$file") ]]; then
+    echo "File found in slow files list. Running once..."
+    num_iterations=1
+  else
+    # shellcheck disable=SC2153
+    num_iterations=$NUM_ITERATIONS
+  fi
+
+  strategy="fast"
+  if [[ " ${HI_RES_STRATEGY_FILES[*]} " =~ $(basename "$file") ]]; then
+    echo "Testing with hi_res strategy"
+    strategy="hi_res"
+  fi
+  if ! response=$(python3 -m "scripts.performance.time_partition" "$file" "$num_iterations" "$strategy"); then
+    echo "error: $response"
+    exit 1
+  fi
+  average_time=$(echo "$response" | awk '/Average time:/ {print $3}')
+  echo "Average execution time: $average_time seconds"
+  echo "$(basename "$file"),$num_iterations,$average_time" >>"$RESULTS_FILE"
+done
+
+# NOTE: Be careful if updating this message. The benchmarking script looks for this message to get the CSV file name.
+echo "Benchmarking completed. Results saved to: $(basename "$RESULTS_FILE")"

scripts/performance/benchmark.sh

@@ -0,0 +1,67 @@
+#!/usr/bin/env bash
+
+# Usage:
+#   - Set the required environment variables (listed below)
+#   - Run the script: ./scripts/performance/benchmark.sh
+
+# Environment Variables:
+#   - DOCKER_TEST: Set to "true" to run benchmark inside a Docker container (default: false)
+#   - NUM_ITERATIONS: Number of iterations for benchmark (e.g., 100) (default: 3)
+#   - INSTANCE_TYPE: Type of benchmark instance (e.g., "c5.xlarge") (default: "unspecified")
+#   - PUBLISH_RESULTS: Set to "true" to publish results to S3 bucket (default: false)
+
+SLOW_FILES=("DA-619p.pdf" "layout-parser-paper-hi_res-16p.pdf" "layout-parser-paper-10p.jpg")
+HI_RES_STRATEGY_FILES=("layout-parser-paper-hi_res-16p.pdf")
+NUM_ITERATIONS=${NUM_ITERATIONS:-2}
+INSTANCE_TYPE=${INSTANCE_TYPE:-"unspecified"}
+
+S3_BUCKET="utic-dev-tech-fixtures"
+S3_RESULTS_DIR="performance-test/results"
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+GIT_HASH="$(git rev-parse --short HEAD)"
+
+# Save the results filename to a temporary file
+RESULTS_FILENAME_FILE=$(mktemp)
+trap 'rm -f $RESULTS_FILENAME_FILE' EXIT
+
+function read_benchmark_logs_for_results() {
+  if [[ $line =~ Results\ saved\ to:\ ([^\ ]+) ]]; then
+    results_filename="${BASH_REMATCH[1]}"
+    echo "CSV file value found: $results_filename"
+    echo "$results_filename" >"$RESULTS_FILENAME_FILE" # Store the value in the temporary file
+  fi
+}
+
+if [[ "$DOCKER_TEST" == "true" ]]; then
+  DOCKER_IMAGE=unstructured:perf-test make docker-build
+  docker rm -f unstructured-perf-test >/dev/null 2>&1
+  docker run \
+    --name unstructured-perf-test \
+    --rm \
+    -e NUM_ITERATIONS="$NUM_ITERATIONS" \
+    -e INSTANCE_TYPE="$INSTANCE_TYPE" \
+    -e GIT_HASH="$GIT_HASH" \
+    -e SLOW_FILES="${SLOW_FILES[*]}" \
+    -e HI_RES_STRATEGY_FILES="${HI_RES_STRATEGY_FILES[*]}" \
+    -v "${SCRIPT_DIR}":/home/notebook-user/scripts/performance \
+    unstructured:perf-test \
+    bash /home/notebook-user/scripts/performance/benchmark-local.sh 2>&1 | tee >(while IFS= read -r line; do
+      read_benchmark_logs_for_results
+    done)
+else
+  NUM_ITERATIONS="$NUM_ITERATIONS" INSTANCE_TYPE="$INSTANCE_TYPE" GIT_HASH="$GIT_HASH" SLOW_FILES="${SLOW_FILES[*]}" HI_RES_STRATEGY_FILES="${HI_RES_STRATEGY_FILES[*]}" "$SCRIPT_DIR"/benchmark-local.sh 2>&1 |
+    tee >(while IFS= read -r line; do
+      read_benchmark_logs_for_results
+    done)
+fi
+# Read the result filename from the temporary file
+results_filename=$(<"$RESULTS_FILENAME_FILE")
+if [[ -z $results_filename ]]; then
+  echo "Error: Results filename value not found in the benchmark logs."
+  exit 1
+fi
+if [[ "$PUBLISH_RESULTS" == "true" ]]; then
+  S3_RESULTS_PATH="$S3_BUCKET/$S3_RESULTS_DIR"
+  echo "Publishing results to S3 bucket: $S3_RESULTS_PATH"
+  aws s3 cp "$SCRIPT_DIR/benchmark_results/$results_filename" "s3://$S3_RESULTS_PATH/"
+fi

scripts/performance/benchmark_partition.py

@@ -0,0 +1,132 @@
+#!/usr/bin/env python3
+"""Measure partition() runtime over a fixed set of representative example-docs files.
+
+Follows the same conventions as the existing scripts/performance tooling:
+    - PDFs and images are run with strategy="hi_res".
+    - Everything else is run with strategy="fast".
+    - Each file is timed over NUM_ITERATIONS runs (after a warmup) and the
+      average is recorded, matching time_partition.py behaviour.
+
+Writes a JSON file mapping each file to its average runtime, plus a ``__total__``
+key with the wall-clock total.  An optional positional argument sets the output
+path (default: scripts/performance/partition-speed-test/benchmark_results.json).
+
+Also writes the total duration to $GITHUB_OUTPUT as ``duration=<seconds>``.
+
+Usage:
+    uv run --no-sync python scripts/performance/benchmark_partition.py [output.json]
+
+Environment variables:
+    NUM_ITERATIONS   number of timed iterations per file (default: 1)
+"""
+
+from __future__ import annotations
+
+import json
+import logging
+import os
+import sys
+import time
+from pathlib import Path
+
+from unstructured.partition.auto import partition
+
+logging.basicConfig(level=logging.INFO, format="%(message)s")
+logger = logging.getLogger(__name__)
+
+
+BENCHMARK_FILES: list[tuple[str, str]] = [
+    # PDFs - hi_res
+    ("example-docs/pdf/a1977-backus-p21.pdf", "hi_res"),
+    ("example-docs/pdf/copy-protected.pdf", "hi_res"),
+    ("example-docs/pdf/reliance.pdf", "hi_res"),
+    ("example-docs/pdf/pdf-with-ocr-text.pdf", "hi_res"),
+    # Images - hi_res
+    ("example-docs/double-column-A.jpg", "hi_res"),
+    ("example-docs/double-column-B.jpg", "hi_res"),
+    ("example-docs/embedded-images-tables.jpg", "hi_res"),
+    # Other document types - fast
+    ("example-docs/contains-pictures.docx", "fast"),
+    ("example-docs/example-10k-1p.html", "fast"),
+    ("example-docs/science-exploration-1p.pptx", "fast"),
+]
+
+NUM_ITERATIONS: int = int(os.environ.get("NUM_ITERATIONS", "1"))
+
+DEFAULT_OUTPUT = Path(__file__).parent / "partition-speed-test" / "benchmark_results.json"
+
+
+def _warmup(filepath: str) -> None:
+    """Run a single fast-strategy partition to warm the process up.
+
+    Mirrors warm_up_process() in time_partition.py: uses a warmup-docs/
+    variant if present, otherwise falls back to the file itself.
+    """
+
+    warmup_dir = Path(__file__).parent / "warmup-docs"
+    warmup_file = warmup_dir / f"warmup{Path(filepath).suffix}"
+    target = str(warmup_file) if warmup_file.exists() else filepath
+    partition(target, strategy="fast")
+
+
+def _measure(filepath: str, strategy: str, iterations: int) -> float:
+    """Return the average wall-clock seconds for partitioning *filepath*.
+
+    Identical logic to time_partition.measure_execution_time().
+    """
+
+    total = 0.0
+    for _ in range(iterations):
+        t0 = time.perf_counter()
+        partition(filepath, strategy=strategy)
+        total += time.perf_counter() - t0
+    return total / iterations
+
+
+def _set_github_output(key: str, value: str) -> None:
+    """Write key=value to $GITHUB_OUTPUT when running in Actions."""
+    gho = os.environ.get("GITHUB_OUTPUT")
+    if gho:
+        with open(gho, "a") as fh:
+            fh.write(f"{key}={value}\n")
+
+
+def main() -> None:
+    output_path = Path(sys.argv[1]) if len(sys.argv) > 1 else DEFAULT_OUTPUT
+    repo_root = Path(__file__).resolve().parent.parent.parent  # scripts/performance/ -> repo root
+
+    logger.info("=" * 60)
+    logger.info(f"Partition benchmark  (NUM_ITERATIONS={NUM_ITERATIONS})")
+    logger.info("=" * 60)
+
+    results: dict[str, float] = {}
+    grand_start = time.perf_counter()
+
+    for rel_path, strategy in BENCHMARK_FILES:
+        filepath = repo_root / rel_path
+        if not filepath.exists():
+            logger.warning(f"  WARNING: {rel_path} not found – skipping.")
+            continue
+
+        logger.info(f"  {rel_path}  (strategy={strategy}, iterations={NUM_ITERATIONS})")
+        _warmup(str(filepath))
+        avg = _measure(str(filepath), strategy, NUM_ITERATIONS)
+        results[rel_path] = round(avg, 4)
+        logger.info(f"    avg {avg:.2f}s")
+
+    total_seconds = round(time.perf_counter() - grand_start, 2)
+    results["__total__"] = total_seconds
+
+    logger.info(f"\nTotal wall-clock time: {total_seconds}s")
+
+    # Write JSON results file (consumed by compare_benchmark.py)
+    output_path.parent.mkdir(parents=True, exist_ok=True)
+    output_path.write_text(json.dumps(results, indent=2) + "\n")
+    logger.info(f"Results written to {output_path}")
+
+    # Also expose total as a GitHub Actions step output
+    _set_github_output("duration", str(int(total_seconds)))
+
+
+if __name__ == "__main__":
+    main()

scripts/performance/compare_benchmark.py

@@ -0,0 +1,284 @@
+#!/usr/bin/env python3
+"""Compare this run's partition() benchmark against a rolling baseline.
+
+Why a rolling baseline (and not a single stored "best"):
+    GitHub's shared ``ubuntu-latest`` runners vary in speed by ~1.6x run-to-run,
+    and that variance scales the whole benchmark (partition work *and* fixed
+    overhead). A single all-time-minimum baseline is therefore unbeatable the
+    moment one lucky-fast runner records it -- every later run on a normal runner
+    exceeds best*(1+threshold) and the check fails for everyone. (That is exactly
+    what happened: a frozen 81s "best" vs a real ~130s fleet.)
+
+    Instead we keep the last ``--window`` runs from ``main`` and compare against
+    their **median**, which a single fast/slow outlier can't poison. The baseline
+    tracks reality over time rather than ratcheting to an unrepeatable minimum.
+
+Storage:
+    Each ``main`` run is one immutable JSON record under ``HISTORY_DIR`` (synced
+    to/from S3 by the workflow). This script never talks to S3 -- it only reads
+    the local history dir and, with ``--record``, writes this run's record there
+    for the workflow to upload. Pruning of old objects is an S3 lifecycle rule.
+
+Usage:
+    uv run --no-sync python scripts/performance/compare_benchmark.py \
+        benchmark_results.json \
+        HISTORY_DIR \
+        [--threshold 0.30] [--window 20] [--min-samples 5] [--record]
+
+    benchmark_results.json  this run's results from benchmark_partition.py
+    HISTORY_DIR             dir of per-run history records (may be empty/missing)
+    --threshold            regression allowance over the median (default 0.30)
+    --window               number of most-recent records to median over (default 20)
+    --min-samples          below this many records, warm-up (record + pass, no gate)
+    --record               write this run's record into HISTORY_DIR (main runs only);
+                           recording happens regardless of pass/fail so the baseline
+                           can't get stuck.
+"""
+
+from __future__ import annotations
+
+import argparse
+import datetime as dt
+import json
+import logging
+import math
+import os
+import statistics
+import sys
+from pathlib import Path
+
+logging.basicConfig(level=logging.INFO, format="%(levelname)s: %(message)s")
+logger = logging.getLogger(__name__)
+
+
+def _github_output(key: str, value: str) -> None:
+    """Write a key=value pair to $GITHUB_OUTPUT when running in Actions."""
+    gho = os.environ.get("GITHUB_OUTPUT")
+    if gho:
+        with open(gho, "a") as fh:
+            fh.write(f"{key}={value}\n")
+
+
+def _fmt(seconds: float) -> str:
+    if seconds is None or math.isnan(seconds):
+        return "    n/a"
+    return f"{seconds:7.2f}s"
+
+
+def _pct_diff(current: float, baseline: float) -> str:
+    if not baseline:
+        return "   n/a"
+    diff = (current - baseline) / baseline * 100
+    sign = "+" if diff >= 0 else ""
+    return f"{sign}{diff:.1f}%"
+
+
+def _runner_info() -> dict:
+    """Best-effort CPU/nproc capture, purely for later visibility of variance."""
+    cpu_model = None
+    try:
+        for line in Path("/proc/cpuinfo").read_text().splitlines():
+            if line.lower().startswith("model name"):
+                cpu_model = line.split(":", 1)[1].strip()
+                break
+    except OSError:
+        pass
+    return {"cpu_model": cpu_model, "nproc": os.cpu_count()}
+
+
+def _is_number(value: object) -> bool:
+    """True for real numeric values; bool is rejected (it's a subclass of int)."""
+    return isinstance(value, (int, float)) and not isinstance(value, bool)
+
+
+def load_history(history_dir: Path) -> list[dict]:
+    """Load every per-run record from HISTORY_DIR, sorted oldest -> newest.
+
+    Records are deduped by sha (keeping the newest per sha by timestamp) so the
+    median is correct even if S3 still holds legacy timestamped objects that share
+    a sha. Records without a sha are kept individually -- they're never collapsed
+    together. Records whose ``total`` is missing or non-numeric are skipped so a
+    malformed object can't crash ``statistics.median``.
+    """
+    if not history_dir.is_dir():
+        return []
+    records: list[dict] = []
+    for f in history_dir.glob("*.json"):
+        try:
+            rec = json.loads(f.read_text())
+        except (json.JSONDecodeError, OSError) as e:
+            logger.warning(f"skipping unreadable history record {f.name}: {e}")
+            continue
+        if isinstance(rec, dict) and _is_number(rec.get("total")):
+            records.append(rec)
+        else:
+            logger.warning(f"skipping history record {f.name}: missing/non-numeric 'total'")
+    records.sort(key=lambda r: r.get("timestamp") or "")
+
+    # Dedupe by sha, keeping the newest record per sha. Records sort oldest->newest
+    # above, so a later same-sha record overwrites the earlier one. Empty/absent sha
+    # is kept per-record (do not collapse all sha-less records together).
+    deduped: list[dict] = []
+    by_sha: dict[str, int] = {}
+    for rec in records:
+        sha = rec.get("sha") or ""
+        if sha and sha in by_sha:
+            deduped[by_sha[sha]] = rec
+        else:
+            if sha:
+                by_sha[sha] = len(deduped)
+            deduped.append(rec)
+    # Overwriting a same-sha record in place keeps the *first* occurrence's slot but
+    # the *newer* record's timestamp, which can leave deduped out of chronological
+    # order. Re-sort so callers can rely on history[-window:] being the newest runs.
+    return sorted(deduped, key=lambda r: r.get("timestamp") or "")
+
+
+def build_record(current: dict) -> dict:
+    """Build this run's history record from results + CI environment metadata."""
+    now = dt.datetime.now(dt.timezone.utc)
+    sha = os.environ.get("GITHUB_SHA", "")
+    return {
+        "sha": sha,
+        "run_id": int(os.environ["GITHUB_RUN_ID"]) if os.environ.get("GITHUB_RUN_ID") else None,
+        "timestamp": now.strftime("%Y-%m-%dT%H:%M:%SZ"),
+        "event": os.environ.get("GITHUB_EVENT_NAME"),
+        "ref": os.environ.get("GITHUB_REF_NAME"),
+        "seed": False,
+        "iterations": int(os.environ.get("NUM_ITERATIONS", "0")) or None,
+        "runner": _runner_info(),
+        "total": round(current["__total__"], 2),
+        "per_file": {k: round(v, 4) for k, v in current.items() if k != "__total__"},
+    }
+
+
+def write_record(history_dir: Path, record: dict) -> Path:
+    """Write the record into HISTORY_DIR, replacing any prior record for this sha.
+
+    The object name is keyed by sha alone (not timestamped), so a re-run of the
+    same commit overwrites the same object. This is what keeps the rolling median
+    from double-counting a commit: the workflow's ``aws s3 sync`` (no ``--delete``)
+    would otherwise leave a stale, differently-named same-sha object behind. The
+    timestamp survives as a record field, which is what load/sort uses.
+    """
+    history_dir.mkdir(parents=True, exist_ok=True)
+    sha = record.get("sha") or ""
+    if sha:
+        name = f"{sha[:12]}.json"
+    else:
+        # No sha (e.g. local/manual): fall back to a timestamp-derived name. Keep it
+        # filesystem-safe by stripping the separators, as the old naming did.
+        name = record["timestamp"].replace("-", "").replace(":", "") + ".json"
+    out = history_dir / name
+    out.write_text(json.dumps(record, indent=2) + "\n")
+    return out
+
+
+def _median_per_file(window: list[dict]) -> dict[str, float]:
+    files: set[str] = set()
+    for r in window:
+        files.update(r.get("per_file", {}).keys())
+    medians: dict[str, float] = {}
+    for fname in files:
+        vals = [
+            r["per_file"][fname] for r in window if _is_number(r.get("per_file", {}).get(fname))
+        ]
+        if vals:
+            medians[fname] = statistics.median(vals)
+    return medians
+
+
+def main() -> None:
+    ap = argparse.ArgumentParser(
+        description=__doc__, formatter_class=argparse.RawDescriptionHelpFormatter
+    )
+    ap.add_argument("current_results", type=Path)
+    ap.add_argument("history_dir", type=Path)
+    ap.add_argument("--threshold", type=float, default=0.30)
+    ap.add_argument("--window", type=int, default=20)
+    ap.add_argument("--min-samples", type=int, default=5)
+    ap.add_argument(
+        "--record", action="store_true", help="write this run into the history (main runs)"
+    )
+    args = ap.parse_args()
+
+    # --window is used as history[-args.window:]; 0 silently means "all history"
+    # ([-0:] == [0:]) and negatives slice from the wrong end. Require >= 1.
+    if args.window < 1:
+        ap.error("--window must be >= 1")
+    if args.min_samples < 0:
+        ap.error("--min-samples must be >= 0")
+
+    current: dict[str, float] = json.loads(args.current_results.read_text())
+    current_total: float = current["__total__"]
+
+    history = load_history(args.history_dir)
+    window = history[-args.window :]
+
+    # Record first (main runs only) so the baseline always reflects reality,
+    # regardless of whether this run passes the gate -- the opposite of the old
+    # min-ratchet, which could only ever lower the bar and so got stuck.
+    if args.record:
+        rec_path = write_record(args.history_dir, build_record(current))
+        logger.info(f"recorded this run -> {rec_path.name}")
+
+    # Warm-up: not enough history to gate yet. Observe and pass. An empty window can
+    # never produce a baseline (statistics.median([]) raises), so it is always warm-up
+    # regardless of --min-samples (which may be 0).
+    if not window or len(window) < args.min_samples:
+        logger.info(
+            f"WARM-UP: {len(window)} of {args.min_samples} baseline samples present "
+            f"-- recording only, not gating. (current total {current_total:.2f}s)"
+        )
+        _github_output("regression", "false")
+        _github_output("warmup", "true")
+        sys.exit(0)
+
+    baseline = statistics.median(r["total"] for r in window)
+    limit = baseline * (1.0 + args.threshold)
+    per_file_baseline = _median_per_file(window)
+
+    all_files = sorted((set(current) | set(per_file_baseline)) - {"__total__"})
+    col_w = max((len(f) for f in all_files), default=40) + 2
+    header = f"{'File':<{col_w}} {'Current':>9}  {'Median':>9}  {'Delta':>8}"
+    logger.info("=" * len(header))
+    logger.info(f"Partition benchmark vs rolling median of last {len(window)} main runs")
+    logger.info("=" * len(header))
+    logger.info(header)
+    logger.info("-" * len(header))
+    for fname in all_files:
+        c = current.get(fname, float("nan"))
+        b = per_file_baseline.get(fname, float("nan"))
+        logger.info(f"{fname:<{col_w}} {_fmt(c)}  {_fmt(b)}  {_pct_diff(c, b):>8}")
+    logger.info("-" * len(header))
+    logger.info(
+        f"{'TOTAL':<{col_w}} {_fmt(current_total)}  {_fmt(baseline)}"
+        f"  {_pct_diff(current_total, baseline):>8}"
+    )
+    logger.info("")
+    logger.info(
+        f"Baseline : median of {len(window)} runs = {baseline:.2f}s  "
+        f"(threshold {args.threshold * 100:.0f}%, fail if > {limit:.2f}s)"
+    )
+    logger.info("")
+
+    if current_total > limit:
+        excess_pct = (current_total - baseline) / baseline * 100
+        logger.error(
+            f"FAIL: current runtime {current_total:.2f}s exceeds the median baseline "
+            f"{baseline:.2f}s by {excess_pct:.1f}% (threshold {args.threshold * 100:.0f}%, "
+            f"limit {limit:.2f}s)."
+        )
+        _github_output("regression", "true")
+        sys.exit(1)
+
+    logger.info(
+        f"PASS: {current_total:.2f}s is within {args.threshold * 100:.0f}% of the "
+        f"median baseline {baseline:.2f}s."
+    )
+    _github_output("regression", "false")
+    sys.exit(0)
+
+
+if __name__ == "__main__":
+    main()

scripts/performance/docs/DA-1p.pdf

@@ -0,0 +1 @@
+../../../example-docs/DA-1p.pdf

scripts/performance/docs/DA-619p.pdf

@@ -0,0 +1 @@
+../../../example-docs/DA-619p.pdf

scripts/performance/docs/book-war-and-peace-1225p.txt

@@ -0,0 +1 @@
+../../../example-docs/book-war-and-peace-1225p.txt

scripts/performance/docs/book-war-and-peace-1p.txt

@@ -0,0 +1 @@
+../../../example-docs/book-war-and-peace-1p.txt

scripts/performance/docs/example-10k-1p.html

@@ -0,0 +1 @@
+../../../example-docs/example-10k-1p.html

scripts/performance/docs/example-10k-230p.html

@@ -0,0 +1 @@
+../../../example-docs/example-10k-230p.html

scripts/performance/docs/handbook-1p.docx

@@ -0,0 +1 @@
+../../../example-docs/handbook-1p.docx

scripts/performance/docs/handbook-872p.docx

@@ -0,0 +1 @@
+../../../example-docs/handbook-872p.docx

scripts/performance/docs/layout-parser-paper-10p.jpg

@@ -0,0 +1 @@
+../../../example-docs/layout-parser-paper-10p.jpg

scripts/performance/docs/layout-parser-paper-1p.jpg

@@ -0,0 +1 @@
+../../../example-docs/layout-parser-paper-fast.jpg

scripts/performance/docs/layout-parser-paper-fast-16p.pdf

@@ -0,0 +1 @@
+../../../example-docs/layout-parser-paper.pdf

scripts/performance/docs/layout-parser-paper-hi_res-16p.pdf

@@ -0,0 +1 @@
+../../../example-docs/layout-parser-paper.pdf

scripts/performance/docs/science-exploration-1p.pptx

@@ -0,0 +1 @@
+../../../example-docs/science-exploration-1p.pptx

scripts/performance/docs/science-exploration-369p.pptx

@@ -0,0 +1 @@
+../../../example-docs/science-exploration-369p.pptx

scripts/performance/get-stats-name.sh

@@ -0,0 +1,21 @@
+#!/usr/bin/env bash
+
+# get a string representing the system stats. we should be able to infer
+# this from aws types, but this guarantees we have the info we need in all cases
+
+# hack to get gpus available for processing
+# assumes nvidia drivers available for inference tasks
+if command -v nvidia-smi &>/dev/null; then
+  gpu=$(nvidia-smi --query-gpu=name --format=csv,noheader | wc -l)
+else
+  gpu="0"
+fi
+if command -v sysctl >/dev/null && command -v system_profiler >/dev/null; then
+  cpu=$(sysctl -n hw.logicalcpu_max)
+  mem=$(sysctl -n hw.memsize | awk '{printf "%.0fGB",$0/1024/1024/1024}')
+else
+  cpu=$(getconf _NPROCESSORS_ONLN)
+  mem=$(grep 'MemTotal' /proc/meminfo | awk '{printf "%.0fGB",$2/1024/1024}')
+fi
+
+echo "${cpu}cpu_${gpu}gpu_${mem}mem"

scripts/performance/profile.sh

@@ -0,0 +1,365 @@
+#!/usr/bin/env bash
+
+# Performance profiling and visualization of code using cProfile and memray.
+
+# Environment Variables:
+#   - DOCKER_TEST: Set to true to run profiling inside a Docker container (default: false)
+
+# Usage:
+# - Run the script and choose the profiling mode: 'run' or 'view'.
+# - In the 'run' mode, you can profile custom files or select existing test files.
+# - In the 'view' mode, you can view previously generated profiling results.
+# - The script supports time profiling with cProfile and memory profiling with memray.
+# - Users can choose different visualization options such as flamegraphs, tables, trees, summaries, and statistics.
+# - Test documents are (optionally) synced from an S3 bucket to a local directory before running the profiles.
+
+# Dependencies:
+# - memray package for memory profiling and visualization.
+# - flameprof and snakeviz for time profiling and visualization.
+# - AWS CLI for syncing files from S3 (if applicable).
+
+# Package dependencies can be installed with `uv pip install -r scripts/performance/requirements.txt`
+
+# Usage example:
+# ./scripts/performance/profile.sh
+
+# NOTE: because memray does not build wheels for ARM-Linux, this script can not run in an ARM Docker container on an M1 Mac (though emulated AMD would work).
+
+# Validate dependencies
+check_python_module() {
+  if ! python3 -c "import $1" >/dev/null 2>&1; then
+    echo "Error: Python module $1 is not installed. Please install required dependencies with 'uv pip install -r scripts/performance/requirements.txt'."
+    exit 1
+  fi
+}
+validate_dependencies() {
+  check_python_module memray
+  check_python_module flameprof
+}
+
+# only validate in non-docker context (since we install dependencies on the fly in docker)
+if [[ "$DOCKER_TEST" != "true" ]]; then
+  validate_dependencies
+fi
+
+SCRIPT_DIR=$(dirname "$0")
+# Convert the relative path to module notation
+MODULE_PATH=${SCRIPT_DIR////.}
+# Remove the leading dot if it exists
+MODULE_PATH=${MODULE_PATH#.}
+# Remove the leading dot if it exists again
+MODULE_PATH=${MODULE_PATH#\.}
+
+PROFILE_RESULTS_DIR="$SCRIPT_DIR/profile_results"
+
+# Create PROFILE_RESULTS_DIR if it doesn't exist
+mkdir -p "$PROFILE_RESULTS_DIR"
+
+if [[ "$DOCKER_TEST" == "true" ]]; then
+  SCRIPT_PARENT_DIR=$(dirname "$(dirname "$(realpath "$0")")")
+  docker run -it --rm -v "$SCRIPT_PARENT_DIR:/home/unstructured/scripts" unstructured:dev /bin/bash -c "
+  cd unstructured/
+  uv pip install -r scripts/performance/requirements.txt
+  echo \"Warming the Docker container by running a small partitioning job..\"
+  python3 -c 'from unstructured.partition.auto import partition; partition(\"'""$SCRIPT_DIR/warmup_docs/warmup.pdf'\", strategy=\"hi_res\")[1]'
+  ./scripts/performance/profile.sh
+  "
+  exit 0
+fi
+
+check_display() {
+  if system_profiler SPDisplaysDataType 2>/dev/null | grep -q "Display Type"; then
+    return 0 # Display is present
+  else
+    return 1 # Display is not present (headless context)
+  fi
+}
+
+view_profile_headless() {
+  # Several of the visualization options require a graphical interface. If DISPLAY is not set, we can't use those options.
+
+  extension=".bin"
+  result_file="${result_file%.*}$extension"
+
+  if [[ ! -f "$result_file" ]]; then
+    unset result_file # Unset the result_file variable to go back to the "Select a file" view
+    echo "Result file not found. Please choose a different profile type or go back."
+  else
+    while true; do
+      read -r -p "Choose visualization type: (1) tree (2) summary (3) stats (b) back, (q) quit: " -n 1 visualization_type
+      echo
+
+      if [[ $visualization_type == "b" ]]; then
+        unset result_file # Unset the result_file variable to go back to the "Select a file" view
+        break
+      elif [[ $visualization_type == "q" ]]; then
+        exit 0
+      fi
+
+      case $visualization_type in
+      "1")
+        python3 -m memray tree "$result_file"
+        ;;
+      "2")
+        python3 -m memray summary "$result_file"
+        ;;
+      "3")
+        python3 -m memray stats "$result_file"
+        ;;
+      *)
+        echo "Invalid visualization type. Please try again."
+        ;;
+      esac
+    done
+  fi
+}
+
+view_profile_with_head() {
+  while true; do
+    read -r -p "Choose profile type: (1) time (2) memory (3) speedscope (b) back, (q) quit: " -n 1 profile_type
+    echo
+
+    if [[ $profile_type == "b" ]]; then
+      unset result_file # Unset the result_file variable to go back to the "Select a file" view
+      break
+    elif [[ $profile_type == "q" ]]; then
+      exit 0
+    fi
+
+    if [[ $profile_type == "1" ]]; then
+      extension=".prof"
+    elif [[ $profile_type == "2" ]]; then
+      extension=".bin"
+    elif [[ $profile_type == "3" ]]; then
+      extension=".speedscope"
+    else
+      echo "Invalid profile type. Please try again."
+      continue
+    fi
+
+    result_file="${result_file%.*}$extension"
+
+    if [[ ! -f "$result_file" ]]; then
+      echo "Result file not found. Please choose a different profile type or go back."
+      continue
+    fi
+
+    if [[ $profile_type == "3" ]]; then
+      speedscope "$result_file"
+    elif [[ $profile_type == "2" ]]; then
+      while true; do
+        read -r -p "Choose visualization type: (1) flamegraph (2) table (3) tree (4) summary (5) stats (b) back, (q) quit: " -n 1 visualization_type
+        echo
+
+        if [[ $visualization_type == "b" ]]; then
+          break
+        elif [[ $visualization_type == "q" ]]; then
+          exit 0
+        fi
+
+        case $visualization_type in
+        "1")
+          rm -f "${result_file}.memray.html"
+          python3 -m memray flamegraph -o "${result_file}.memray.html" "$result_file"
+          open "${result_file}.memray.html"
+          ;;
+        "2")
+          rm -f "${result_file}.table.html"
+          python3 -m memray table -o "${result_file}.table.html" "$result_file"
+          open "${result_file}.table.html"
+          ;;
+        "3")
+          python3 -m memray tree "$result_file"
+          ;;
+        "4")
+          python3 -m memray summary "$result_file"
+          ;;
+        "5")
+          python3 -m memray stats "$result_file"
+          ;;
+        *)
+          echo "Invalid visualization type. Please try again."
+          ;;
+        esac
+      done
+    else
+      while true; do
+        read -r -p "Choose visualization type: (1) flamegraph (2) snakeviz (b) back, (q) quit: " -n 1 visualization_type
+        echo
+
+        if [[ $visualization_type == "b" ]]; then
+          break
+        elif [[ $visualization_type == "q" ]]; then
+          exit 0
+        fi
+
+        case $visualization_type in
+        "1")
+          flameprof_file="${result_file}.flameprof.svg"
+          rm -f "$flameprof_file"
+          python3 -m flameprof "$result_file" >"$flameprof_file"
+          open "$flameprof_file"
+          ;;
+        "2")
+          snakeviz "$result_file"
+          ;;
+        *)
+          echo "Invalid visualization type. Please try again."
+          ;;
+        esac
+      done
+    fi
+
+    break # Return to the beginning
+  done
+}
+
+view_profile() {
+
+  if [ -n "$1" ]; then
+    result_file="$1"
+  fi
+  while true; do
+    if [[ -z $result_file ]]; then
+      echo "Available result files:"
+      result_files=("$PROFILE_RESULTS_DIR"/*.bin)
+      if [[ ${#result_files[@]} -eq 0 ]]; then
+        echo "No result files found."
+        return
+      fi
+
+      for ((i = 0; i < ${#result_files[@]}; i++)); do
+        filename="${result_files[$i]##*/}"
+        filename="${filename%.*}"
+        echo "$i. $filename"
+      done
+
+      read -r -p "Enter the number corresponding to the result file you want to view (b to go back, q to quit): " selection
+      if [[ $selection == "b" ]]; then
+        return
+      elif [[ $selection == "q" ]]; then
+        exit 0
+      fi
+
+      result_file="${result_files[$selection]}"
+    fi
+
+    if check_display; then
+      view_profile_with_head "$result_file"
+    else
+      view_profile_headless "$result_file"
+    fi
+  done
+}
+
+run_profile() {
+  while true; do
+    read -r -p "Choose an option: 1) Existing test file, (2) Custom file, (b) back, (q) quit: " -n 1 option
+    echo
+
+    if [[ $option == "b" ]]; then
+      return
+    elif [[ $option == "q" ]]; then
+      exit 0
+    fi
+
+    if [[ $option == "1" ]]; then
+      echo "Available test files:"
+      test_files=("$SCRIPT_DIR/docs"/*)
+      if [[ ${#test_files[@]} -eq 0 ]]; then
+        echo "No test files found."
+        return
+      fi
+
+      for ((i = 0; i < ${#test_files[@]}; i++)); do
+        echo "$i. ${test_files[$i]}"
+      done
+
+      read -r -p "Enter the number corresponding to the test file you want to run followed by return (b to go back, q to quit): " selection
+      if [[ $selection == "b" ]]; then
+        return
+      elif [[ $selection == "q" ]]; then
+        exit 0
+      fi
+
+      test_file="${test_files[$selection]}"
+    elif [[ $option == "2" ]]; then
+      read -r -p "Enter the path to the custom file: " test_file
+    else
+      echo "Invalid option. Please try again."
+      continue
+    fi
+
+    # Delete the output files if they exist
+    rm -f "$PROFILE_RESULTS_DIR/${test_file##*/}.prof"
+    rm -f "$PROFILE_RESULTS_DIR/${test_file##*/}.bin"
+
+    # Pick the strategy
+    while true; do
+      read -r -p "Choose a strategy: 1) auto, (2) fast, (3) hi_res, (4) ocr_only (b) back, (q) quit: " -n 1 strategy_option
+      echo
+
+      if [[ $strategy_option == "b" ]]; then
+        return
+      elif [[ $strategy_option == "q" ]]; then
+        exit 0
+      fi
+
+      case $strategy_option in
+      "1")
+        strategy="auto"
+        break
+        ;;
+      "2")
+        strategy="fast"
+        break
+        ;;
+      "3")
+        strategy="hi_res"
+        break
+        ;;
+      "4")
+        strategy="ocr_only"
+        break
+        ;;
+      *)
+        echo "Invalid strategy option. Please try again."
+        ;;
+      esac
+    done
+
+    echo "Running time profile..."
+    python3 -m cProfile -s cumulative -o "$PROFILE_RESULTS_DIR/${test_file##*/}.prof" -m "$MODULE_PATH.run_partition" "$test_file" "$strategy"
+    echo "Running memory profile..."
+    python3 -m memray run -o "$PROFILE_RESULTS_DIR/${test_file##*/}.bin" -m "$MODULE_PATH.run_partition" "$test_file" "$strategy"
+    echo "Running py-spy for detailed run time profiling (this can take some time)..."
+    py-spy record --subprocesses -i -o "$PROFILE_RESULTS_DIR/${test_file##*/}.speedscope" --format speedscope -- python3 -m "$MODULE_PATH.run_partition" "$test_file" "$strategy"
+    echo "Profiling completed."
+    echo "Viewing results for $test_file"
+    echo "The py-spy produced speedscope profile can be viewed on https://www.speedscope.app or locally by installing via 'npm install -g speedscope'"
+    result_file=$PROFILE_RESULTS_DIR/$(basename "$test_file")
+    view_profile "${result_file}.bin" # Go directly to view mode
+  done
+}
+
+while true; do
+  if [[ -n "$1" ]]; then
+    mode="$1"
+  fi
+
+  if [[ -z $result_file ]]; then
+    read -r -p "Choose mode: (1) run, (2) view, (q) quit: " -n 1 mode
+    echo
+  fi
+
+  if [[ $mode == "1" ]]; then
+    run_profile
+  elif [[ $mode == "2" ]]; then
+    unset result_file # Unset the result_file variable before entering the "View" mode
+    view_profile
+  elif [[ $mode == "q" ]]; then
+    exit 0
+  else
+    echo "Invalid mode. Please choose 'view', 'run', or 'quit'."
+  fi
+done

scripts/performance/quick_partition_bench.py

@@ -0,0 +1,266 @@
+"""Quick local benchmark for PDF partition cold/warm timing.
+
+Examples:
+  uv run --active --frozen --no-sync scripts/performance/quick_partition_bench.py \
+    --pdf example-docs/pdf/DA-1p.pdf --strategy fast --repeats 4 --warmups 1 --mode both
+
+  uv run --active --frozen --no-sync scripts/performance/quick_partition_bench.py \
+    --pdf example-docs/pdf/DA-1p.pdf --pdf example-docs/pdf/chevron-page.pdf \
+    --strategy hi_res --repeats 3 --warmups 1 --mode both
+"""
+
+import argparse
+import io
+import json
+import statistics
+import subprocess
+import sys
+import time
+from contextlib import redirect_stderr, redirect_stdout
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[2]
+if str(REPO_ROOT) not in sys.path:
+    sys.path.insert(0, str(REPO_ROOT))
+
+
+def _partition_once(pdf: str, strategy: str) -> dict[str, object]:
+    sink_out = io.StringIO()
+    sink_err = io.StringIO()
+    start = time.perf_counter()
+    try:
+        from unstructured.partition.auto import partition
+
+        with redirect_stdout(sink_out), redirect_stderr(sink_err):
+            elements = partition(filename=pdf, strategy=strategy)
+        return {
+            "ok": True,
+            "elapsed_s": time.perf_counter() - start,
+            "elements": len(elements),
+        }
+    except Exception as exc:  # noqa: BLE001
+        return {"ok": False, "error": f"{type(exc).__name__}: {exc}"}
+
+
+def _summary(values: list[float]) -> dict[str, float]:
+    return {
+        "mean_s": statistics.mean(values),
+        "median_s": statistics.median(values),
+        "min_s": min(values),
+        "max_s": max(values),
+        "stdev_s": statistics.stdev(values) if len(values) > 1 else 0.0,
+    }
+
+
+def _run_cold(pdf: str, strategy: str, repeats: int) -> tuple[list[float], int, list[str]]:
+    times: list[float] = []
+    elements = -1
+    errors: list[str] = []
+
+    for _ in range(repeats):
+        proc = subprocess.run(
+            [
+                sys.executable,
+                __file__,
+                "--_child",
+                "--pdf",
+                pdf,
+                "--strategy",
+                strategy,
+            ],
+            capture_output=True,
+            text=True,
+            check=False,
+        )
+
+        lines = [line.strip() for line in (proc.stdout or "").splitlines() if line.strip()]
+        json_line = next((line for line in reversed(lines) if line.startswith("{")), "")
+        if not json_line:
+            stderr_tail = (proc.stderr or "").strip().splitlines()
+            detail = stderr_tail[-1] if stderr_tail else "no json output"
+            errors.append(f"child failed rc={proc.returncode} ({detail})")
+            continue
+
+        row = json.loads(json_line)
+        if bool(row.get("ok")):
+            times.append(float(row["elapsed_s"]))
+            elements = int(row["elements"])
+        else:
+            errors.append(str(row.get("error", "unknown error")))
+
+    return times, elements, errors
+
+
+def _run_warm(
+    pdf: str,
+    strategy: str,
+    repeats: int,
+    warmups: int,
+) -> tuple[list[float], int, list[str]]:
+    errors: list[str] = []
+
+    for _ in range(warmups):
+        row = _partition_once(pdf=pdf, strategy=strategy)
+        if not bool(row.get("ok")):
+            errors.append(str(row.get("error", "unknown error")))
+            return [], -1, errors
+
+    times: list[float] = []
+    elements = -1
+    for _ in range(repeats):
+        row = _partition_once(pdf=pdf, strategy=strategy)
+        if bool(row.get("ok")):
+            times.append(float(row["elapsed_s"]))
+            elements = int(row["elements"])
+        else:
+            errors.append(str(row.get("error", "unknown error")))
+
+    return times, elements, errors
+
+
+def _collect_pdfs(pdf_args: list[str], pdf_dir_args: list[str]) -> list[str]:
+    paths = [str(Path(p)) for p in pdf_args]
+
+    for pdf_dir in pdf_dir_args:
+        root = Path(pdf_dir)
+        if not root.is_dir():
+            raise FileNotFoundError(f"pdf-dir does not exist: {root}")
+        paths.extend(str(p) for p in sorted(root.rglob("*.pdf")))
+
+    if not paths:
+        raise ValueError("Provide at least one --pdf or --pdf-dir")
+
+    deduped = [Path(p) for p in dict.fromkeys(paths)]
+    missing = [str(p) for p in deduped if not p.exists()]
+    if missing:
+        raise FileNotFoundError(f"Missing files: {', '.join(missing)}")
+
+    return [str(p) for p in deduped]
+
+
+def _print_mode(label: str, values: list[float], elements: int, errors: list[str]) -> None:
+    if not values:
+        first = errors[0] if errors else "unknown error"
+        print(f"  {label} FAILED ({first})", flush=True)
+        return
+
+    s = _summary(values)
+    print(
+        f"  {label} mean={s['mean_s']:.4f}s median={s['median_s']:.4f}s "
+        f"min={s['min_s']:.4f}s max={s['max_s']:.4f}s n={len(values)} elements={elements}",
+        flush=True,
+    )
+    if errors:
+        print(f"  {label} partial_failures={len(errors)} first_error={errors[0]}", flush=True)
+
+
+def main() -> None:
+    parser = argparse.ArgumentParser(description="Quick local PDF partition benchmark")
+    parser.add_argument("--pdf", action="append", default=[], help="PDF path (repeatable)")
+    parser.add_argument("--pdf-dir", action="append", default=[], help="Directory of PDFs")
+    parser.add_argument("--strategy", default="fast", choices=["fast", "hi_res", "auto"])
+    parser.add_argument("--repeats", type=int, default=3)
+    parser.add_argument("--warmups", type=int, default=1)
+    parser.add_argument("--mode", default="both", choices=["cold", "warm", "both"])
+    parser.add_argument("--json-out", default="", help="Optional JSON output path")
+    parser.add_argument("--_child", action="store_true", help=argparse.SUPPRESS)
+    args = parser.parse_args()
+
+    if args._child:
+        print(json.dumps(_partition_once(args.pdf[0], args.strategy)), flush=True)
+        return
+
+    pdfs = _collect_pdfs(args.pdf, args.pdf_dir)
+
+    print(
+        f"strategy={args.strategy} mode={args.mode} repeats={args.repeats} "
+        f"warmups={args.warmups} pdf_count={len(pdfs)}",
+        flush=True,
+    )
+
+    by_mode_times: dict[str, list[float]] = {"cold": [], "warm": []}
+    by_mode_file_means: dict[str, list[float]] = {"cold": [], "warm": []}
+    by_mode_failed_files: dict[str, int] = {"cold": 0, "warm": 0}
+    results: list[dict[str, object]] = []
+
+    for pdf in pdfs:
+        row: dict[str, object] = {"pdf": pdf}
+        print(f"FILE {pdf}", flush=True)
+
+        if args.mode in ("cold", "both"):
+            times, elements, errors = _run_cold(pdf, args.strategy, args.repeats)
+            _print_mode("cold", times, elements, errors)
+            if times:
+                by_mode_times["cold"].extend(times)
+                by_mode_file_means["cold"].append(statistics.mean(times))
+                row["cold"] = {"ok": True, "times_s": times, "elements": elements, "errors": errors}
+            else:
+                by_mode_failed_files["cold"] += 1
+                row["cold"] = {"ok": False, "errors": errors}
+
+        if args.mode in ("warm", "both"):
+            times, elements, errors = _run_warm(pdf, args.strategy, args.repeats, args.warmups)
+            _print_mode("warm", times, elements, errors)
+            if times:
+                by_mode_times["warm"].extend(times)
+                by_mode_file_means["warm"].append(statistics.mean(times))
+                row["warm"] = {
+                    "ok": True,
+                    "times_s": times,
+                    "elements": elements,
+                    "errors": errors,
+                    "warmups": args.warmups,
+                }
+            else:
+                by_mode_failed_files["warm"] += 1
+                row["warm"] = {"ok": False, "errors": errors, "warmups": args.warmups}
+
+        results.append(row)
+
+    aggregate: dict[str, object] = {}
+    print("AGGREGATE", flush=True)
+    for mode in ("cold", "warm"):
+        times = by_mode_times[mode]
+        if not times:
+            continue
+        s = _summary(times)
+        file_mean = statistics.mean(by_mode_file_means[mode])
+        succeeded = len(by_mode_file_means[mode])
+        failed = by_mode_failed_files[mode]
+        aggregate[mode] = {
+            "summary": s,
+            "file_mean_s": file_mean,
+            "samples": len(times),
+            "succeeded_files": succeeded,
+            "failed_files": failed,
+        }
+        print(
+            f"  {mode} succeeded_files={succeeded} failed_files={failed} "
+            f"file_mean={file_mean:.4f}s mean={s['mean_s']:.4f}s median={s['median_s']:.4f}s "
+            f"min={s['min_s']:.4f}s max={s['max_s']:.4f}s "
+            f"stdev={s['stdev_s']:.4f}s samples={len(times)}",
+            flush=True,
+        )
+
+    if args.json_out:
+        out = Path(args.json_out)
+        out.parent.mkdir(parents=True, exist_ok=True)
+        out.write_text(
+            json.dumps(
+                {
+                    "strategy": args.strategy,
+                    "mode": args.mode,
+                    "repeats": args.repeats,
+                    "warmups": args.warmups,
+                    "pdf_count": len(pdfs),
+                    "per_file": results,
+                    "aggregate": aggregate,
+                },
+                indent=2,
+            )
+        )
+        print(f"json_out={out}", flush=True)
+
+
+if __name__ == "__main__":
+    main()

scripts/performance/requirements.txt

@@ -0,0 +1,4 @@
+flameprof>=0.4
+memray>=1.7.0
+snakeviz>=2.2.0
+py-spy>=0.3.14

scripts/performance/run_partition.py

@@ -0,0 +1,19 @@
+import os
+import sys
+
+from unstructured.partition.auto import partition
+
+if __name__ == "__main__":
+    if len(sys.argv) < 3:
+        print(
+            "Please provide the path to the file as the first argument and the strategy as the "
+            "second argument.",
+        )
+        sys.exit(1)
+
+    file_path = sys.argv[1]
+    strategy = sys.argv[2]
+    model_name = sys.argv[3] if len(sys.argv) > 3 else os.environ.get("PARTITION_MODEL_NAME")
+    result = partition(file_path, strategy=strategy, model_name=model_name)
+    # access element in the return value to make sure we got something back, otherwise error
+    result[1]

scripts/performance/time_partition.py

@@ -0,0 +1,38 @@
+import os
+import sys
+import time
+
+from unstructured.partition.auto import partition
+
+
+def warm_up_process(filename):
+    warmup_dir = os.path.join(os.path.dirname(__file__), "warmup-docs")
+    warmup_file = os.path.join(warmup_dir, f"warmup{os.path.splitext(filename)[1]}")
+
+    if os.path.exists(warmup_file):
+        partition(warmup_file, strategy="fast")
+    else:
+        partition(filename, strategy="fast")
+
+
+def measure_execution_time(filename, iterations, strategy):
+    total_time = 0.0
+
+    for _ in range(iterations):
+        start_time = time.time()
+        partition(filename, strategy=strategy)
+        end_time = time.time()
+        execution_time = end_time - start_time
+        total_time += execution_time
+
+    average_time = total_time / iterations
+    print("Average time:", average_time)
+
+
+if __name__ == "__main__":
+    filename = sys.argv[1]
+    iterations = int(sys.argv[2])
+    strategy = sys.argv[3]
+
+    warm_up_process(filename)
+    measure_execution_time(filename, iterations, strategy)

scripts/performance/warmup_docs/warmup.docx

@@ -0,0 +1 @@
+../../../example-docs/handbook-1p.docx

scripts/performance/warmup_docs/warmup.html

@@ -0,0 +1 @@
+../../../example-docs/example-10k-1p.html

scripts/performance/warmup_docs/warmup.jpg

@@ -0,0 +1 @@
+../../../example-docs/layout-parser-paper-fast.jpg

scripts/performance/warmup_docs/warmup.pdf

@@ -0,0 +1 @@
+../../../example-docs/layout-parser-paper-fast.pdf

scripts/performance/warmup_docs/warmup.pptx

@@ -0,0 +1 @@
+../../../example-docs/science-exploration-1p.pptx

scripts/performance/warmup_docs/warmup.txt

@@ -0,0 +1 @@
+../../../example-docs/book-war-and-peace-1p.txt

scripts/renovate-security-bump.sh

@@ -0,0 +1,202 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+# Shared script for Renovate to bump version and update CHANGELOG
+# Can be downloaded and executed in any repo with version and changelog files
+# Auto-detects changed dependencies from git diff
+
+# Use current working directory as repo root (where Renovate executes the script)
+# Override these via environment variables if your repo has different paths
+REPO_ROOT="${REPO_ROOT:-$(pwd)}"
+VERSION_FILE="${VERSION_FILE:-$REPO_ROOT/unstructured/__version__.py}"
+CHANGELOG_FILE="${CHANGELOG_FILE:-$REPO_ROOT/CHANGELOG.md}"
+
+echo "=== Renovate Security Version Bump ==="
+
+# Read current version from __version__.py
+CURRENT_VERSION=$(grep -o -E "(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-dev[0-9]+)?" "$VERSION_FILE")
+echo "Current version: $CURRENT_VERSION"
+
+# Determine release version based on current version format
+if [[ "$CURRENT_VERSION" =~ ^([0-9]+)\.([0-9]+)\.([0-9]+)(-dev.*)?$ ]]; then
+  MAJOR="${BASH_REMATCH[1]}"
+  MINOR="${BASH_REMATCH[2]}"
+  PATCH="${BASH_REMATCH[3]}"
+  DEV_SUFFIX="${BASH_REMATCH[4]}"
+
+  if [[ -n "$DEV_SUFFIX" ]]; then
+    # Strip -dev suffix to release current version
+    RELEASE_VERSION="$MAJOR.$MINOR.$PATCH"
+    echo "Stripping dev suffix: $CURRENT_VERSION → $RELEASE_VERSION"
+  else
+    # Already a release version, bump to next patch
+    NEW_PATCH=$((PATCH + 1))
+    RELEASE_VERSION="$MAJOR.$MINOR.$NEW_PATCH"
+    echo "Bumping patch version: $CURRENT_VERSION → $RELEASE_VERSION"
+  fi
+else
+  echo "Error: Could not parse version: $CURRENT_VERSION"
+  exit 1
+fi
+
+# Update __version__.py
+echo "Updating $VERSION_FILE to version $RELEASE_VERSION"
+
+# Detect quote style used in the file
+if grep -q "__version__ = ['\"]" "$VERSION_FILE"; then
+  if grep -q "__version__ = \"" "$VERSION_FILE"; then
+    # Double quotes
+    sed -i.bak -E "s/__version__ = \"[^\"]+\"/__version__ = \"$RELEASE_VERSION\"/" "$VERSION_FILE"
+  else
+    # Single quotes
+    sed -i.bak -E "s/__version__ = '[^']+'/__version__ = '$RELEASE_VERSION'/" "$VERSION_FILE"
+  fi
+else
+  echo "Error: Could not detect quote style in $VERSION_FILE"
+  exit 1
+fi
+
+# Verify the update succeeded
+if ! grep -q "__version__ = ['\"]${RELEASE_VERSION}['\"]" "$VERSION_FILE"; then
+  echo "Error: Failed to update version in $VERSION_FILE"
+  exit 1
+fi
+
+rm -f "$VERSION_FILE.bak"
+
+# Detect changed packages from git diff (best effort, not critical)
+echo "Detecting changed dependencies..."
+CHANGED_PACKAGES=$(git diff --cached uv.lock pyproject.toml 2>/dev/null | grep -E "^[-+](name|version) = " | sed 's/^[+-]//' | sort -u | head -20 || true)
+
+if [ -z "$CHANGED_PACKAGES" ]; then
+  # Try without --cached
+  CHANGED_PACKAGES=$(git diff uv.lock pyproject.toml 2>/dev/null | grep -E "^[-+](name|version) = " | sed 's/^[+-]//' | sort -u | head -20 || true)
+fi
+
+# Build changelog entry (generic for now, can be manually edited)
+if [ -n "$CHANGED_PACKAGES" ]; then
+  PACKAGE_COUNT=$(echo "$CHANGED_PACKAGES" | wc -l | tr -d ' ')
+  echo "Found $PACKAGE_COUNT changed package(s)"
+  CHANGELOG_ENTRY="- **Security update**: Bumped dependencies to address security vulnerabilities"
+else
+  echo "Could not auto-detect packages, using generic entry"
+  CHANGELOG_ENTRY="- **Security update**: Bumped dependencies to address security vulnerabilities"
+fi
+echo "Changelog entry: $CHANGELOG_ENTRY"
+
+# Update CHANGELOG.md
+echo "Updating CHANGELOG..."
+
+# Only look for -dev version to rename if CURRENT_VERSION had -dev suffix
+if [[ -n "$DEV_SUFFIX" ]]; then
+  # Look for -dev version header in CHANGELOG that matches our version
+  DEV_VERSION_HEADER=$(grep -m 1 -F "## $CURRENT_VERSION" "$CHANGELOG_FILE" || true)
+
+  if [[ -n "$DEV_VERSION_HEADER" ]]; then
+    echo "Found dev version in CHANGELOG: $DEV_VERSION_HEADER"
+
+    # Extract the -dev version number from header
+    DEV_VERSION=$(echo "$DEV_VERSION_HEADER" | grep -o -E "[0-9]+\.[0-9]+\.[0-9]+-dev[0-9]*")
+
+    echo "Renaming CHANGELOG header: $DEV_VERSION → $RELEASE_VERSION"
+
+    # Create awk script to:
+    # 1. Rename the -dev version header
+    # 2. Find or create Fixes section
+    # 3. Append security entry
+    awk -v dev_version="$DEV_VERSION" \
+      -v release_version="$RELEASE_VERSION" \
+      -v security_entry="$CHANGELOG_ENTRY" '
+      BEGIN {
+        in_target_version = 0
+        found_fixes = 0
+        added_entry = 0
+      }
+
+      # Match the dev version header and rename it
+      /^## / {
+        if ($0 ~ "^## " dev_version) {
+          print "## " release_version
+          in_target_version = 1
+          next
+        } else {
+          # Hit a different version header, stop being in target version
+          if (in_target_version && !found_fixes && !added_entry) {
+            # We never found Fixes section, add it before this new version
+            print ""
+            print "### Fixes"
+            print security_entry
+            print ""
+            added_entry = 1
+          }
+          in_target_version = 0
+          found_fixes = 0
+        }
+      }
+
+      # Found Fixes section in target version
+      /^### Fixes/ && in_target_version {
+        print
+        print security_entry
+        found_fixes = 1
+        added_entry = 1
+        next
+      }
+
+      { print }
+
+      END {
+        # Handle case where target dev version is last entry and has no Fixes section
+        if (in_target_version && !found_fixes && !added_entry) {
+          print ""
+          print "### Fixes"
+          print security_entry
+        }
+      }
+    ' "$CHANGELOG_FILE" >"$CHANGELOG_FILE.tmp"
+
+    mv "$CHANGELOG_FILE.tmp" "$CHANGELOG_FILE"
+  else
+    # Dev version in __version__.py but no dev header found in CHANGELOG
+    # This shouldn't happen, but create new entry as fallback
+    echo "Warning: Current version has -dev suffix but no matching dev header in CHANGELOG"
+    echo "Creating new entry for $RELEASE_VERSION"
+
+    cat >/tmp/new_changelog_section.tmp <<EOF
+## $RELEASE_VERSION
+
+### Fixes
+$CHANGELOG_ENTRY
+
+EOF
+
+    cat /tmp/new_changelog_section.tmp "$CHANGELOG_FILE" >"$CHANGELOG_FILE.tmp"
+    mv "$CHANGELOG_FILE.tmp" "$CHANGELOG_FILE"
+    rm -f /tmp/new_changelog_section.tmp
+  fi
+else
+  # Current version was already a release, so we bumped to next patch
+  # Create new release entry at top
+  echo "Current version was already released, creating new entry for $RELEASE_VERSION"
+
+  cat >/tmp/new_changelog_section.tmp <<EOF
+## $RELEASE_VERSION
+
+### Fixes
+$CHANGELOG_ENTRY
+
+EOF
+
+  cat /tmp/new_changelog_section.tmp "$CHANGELOG_FILE" >"$CHANGELOG_FILE.tmp"
+  mv "$CHANGELOG_FILE.tmp" "$CHANGELOG_FILE"
+  rm -f /tmp/new_changelog_section.tmp
+fi
+
+echo ""
+echo "✓ Successfully updated version to $RELEASE_VERSION"
+echo "✓ Updated CHANGELOG with security fix entry"
+echo ""
+echo "Modified files:"
+echo "  - $VERSION_FILE"
+echo "  - $CHANGELOG_FILE"

scripts/setup_al2.sh

@@ -0,0 +1,112 @@
+#!/usr/bin/env bash
+set +u
+
+if [ -z "$1" ]; then
+  echo "When running this script, please supply the name of the user account for which to set up unstructured dependencies."
+  echo "Ex: ${0} abertl"
+  exit 1
+fi
+
+set -eux
+
+# Set package manager command for this distribution
+pac="yum"
+
+# If we're not running as root, we want to prefix certain commands with sudo
+if [[ $(whoami) == 'root' ]]; then
+  $pac update -y
+  $pac install -y sudo
+  sudo=''
+else
+  type -p sudo >/dev/null || (echo "Please have an administrator install sudo and add you to the sudo group before continuing." && exit 1)
+  sudo='sudo'
+fi
+
+# Set user account for which we're configuring the tools
+USER_ACCOUNT=$1
+
+# Update existing packages
+$sudo $pac update -y
+
+#### Utils
+# Prerequisites
+$sudo $pac install -y gcc wget tar curl make xz-devel
+# Install non-ancient version of sed
+wget http://ftp.gnu.org/gnu/sed/sed-4.9.tar.gz
+tar xvf sed-4.9.tar.gz
+cd sed-4.9/
+./configure && make && $sudo make install
+cd ..
+
+#### Git
+# Install git
+$sudo $pac install -y git
+
+#### Python
+# Install tools needed to build python
+$sudo $pac install -y bzip2 sqlite zlib-devel readline-devel sqlite-devel openssl-devel tk-devel libffi-devel bzip2-devel
+# Install pyenv
+sudo -u "$USER_ACCOUNT" -i <<'EOF'
+    if [[ ! -d "$HOME"/.pyenv ]]; then
+        cd $HOME
+        curl https://pyenv.run | bash
+        touch "$HOME"/.bashrc
+        # Remove initialization lines from .bashrc if they are already there, so we don't duplicate them
+        # shellcheck disable=SC2016
+        sed -i '/export PYENV_ROOT="$HOME\/.pyenv"/d' "$HOME"/.bashrc
+        # shellcheck disable=SC2016
+        sed -i '/command -v pyenv >\/dev\/null || export PATH="$PYENV_ROOT\/bin:$PATH"/d' "$HOME"/.bashrc
+        # shellcheck disable=SC2016
+        sed -i '/eval "$(pyenv init -)"/d' "$HOME"/.bashrc
+        # shellcheck disable=SC2016
+        sed -i '/eval "$(pyenv virtualenv-init -)"/d' "$HOME"/.bashrc
+        # Add initialization lines to .bashrc
+        # shellcheck disable=SC2016
+        cat <<'EOT' | cat - "$HOME"/.bashrc > temp && mv temp "$HOME"/.bashrc
+export PYENV_ROOT="$HOME/.pyenv"
+command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+EOT
+        # install python
+        source "$HOME"/.bashrc
+        pyenv install 3.12
+    fi
+EOF
+
+#### OpenCV dependencies
+$sudo $pac install -y mesa-libGL
+
+#### Poppler
+# Install poppler
+$sudo $pac install -y poppler-utils
+
+#### Tesseract
+# Install dependencies for image and pdf manipulation
+$sudo $pac install -y opencv opencv-devel opencv-python perl-core clang libpng-devel libtiff-devel libwebp-devel libjpeg-turbo-devel git-core libtool pkgconfig xz
+# Install leptonica (tesseract dependency)
+wget https://github.com/DanBloomberg/leptonica/releases/download/1.75.1/leptonica-1.75.1.tar.gz
+tar -xzvf leptonica-1.75.1.tar.gz
+cd leptonica-1.75.1
+./configure && make && $sudo make install
+cd ..
+# Install autoconf-archive (tesseract dependency)
+wget http://mirror.squ.edu.om/gnu/autoconf-archive/autoconf-archive-2017.09.28.tar.xz
+tar -xvf autoconf-archive-2017.09.28.tar.xz
+cd autoconf-archive-2017.09.28
+./configure && make && $sudo make install
+$sudo cp m4/* /usr/share/aclocal
+cd ..
+# Install tesseract
+git clone --depth 1 https://github.com/tesseract-ocr/tesseract.git tesseract-ocr
+cd tesseract-ocr
+export PKG_CONFIG_PATH=/usr/local/lib/pkgconfig
+./autogen.sh
+./configure && make && $sudo make install
+cd ..
+# Install tesseract languages
+git clone https://github.com/tesseract-ocr/tessdata.git
+$sudo cp tessdata/*.traineddata /usr/local/share/tessdata
+
+#### libmagic
+$sudo $pac install -y file-devel

scripts/setup_ubuntu.sh

@@ -0,0 +1,100 @@
+#!/usr/bin/env bash
+set +u
+
+if [ -z "$1" ]; then
+  echo "When running this script, please supply the name of the user account for which to set up unstructured dependencies."
+  echo "Ex: ${0} abertl"
+  exit 1
+fi
+
+set -eux
+
+# Set package manager command for this distribution
+pac="apt"
+
+# If we're not running as root, we want to prefix certain commands with sudo
+if [[ $(whoami) == 'root' ]]; then
+  $pac update -y
+  $pac install -y sudo
+  sudo=''
+else
+  type -p sudo >/dev/null || (echo "Please have an administrator install sudo and add you to the sudo group before continuing." && exit 1)
+  sudo='sudo'
+fi
+
+# Set user account for which we're configuring the tools
+USER_ACCOUNT=$1
+
+# Update existing packages
+# Reconfigure the service that detects the need for service restarts from interactive mode (user
+# needs to manually confirm which services to restart) to automatic. If we don't do this we'll
+# get hung up on a screen asking us which services we want to restart after upgrading packages.
+$sudo $pac update -y
+if [[ -d /etc/needrestart/conf.d ]]; then
+  # shellcheck disable=SC2016
+  echo '$nrconf{restart} = '"'a';" | $sudo tee /etc/needrestart/conf.d/99z_temp_disable.conf
+fi
+$sudo $pac upgrade -y
+
+#### Utils
+# Prerequisites
+$sudo env DEBIAN_FRONTEND="noninteractive" $pac install -y gcc wget tar curl make xz-utils build-essential tzdata rsync
+
+#### Git
+# Install git
+$sudo $pac install -y git
+
+#### Python
+# Install tools needed to build python
+$sudo $pac install -y bzip2 sqlite zlib1g-dev libreadline-dev libsqlite3-dev libssl-dev tk-dev libffi-dev libbz2-dev llvm libncursesw5-dev libxml2-dev libxmlsec1-dev liblzma-dev
+# Install pyenv
+sudo -u "$USER_ACCOUNT" -i <<'EOF'
+    if [[ ! -d "$HOME"/.pyenv ]]; then
+        cd $HOME
+        curl https://pyenv.run | bash
+        touch "$HOME"/.bashrc
+        # Remove initialization lines from .bashrc if they are already there, so we don't duplicate them
+        # shellcheck disable=SC2016
+        sed -i '/export PYENV_ROOT="$HOME\/.pyenv"/d' "$HOME"/.bashrc
+        # shellcheck disable=SC2016
+        sed -i '/command -v pyenv >\/dev\/null || export PATH="$PYENV_ROOT\/bin:$PATH"/d' "$HOME"/.bashrc
+        # shellcheck disable=SC2016
+        sed -i '/eval "$(pyenv init -)"/d' "$HOME"/.bashrc
+        # shellcheck disable=SC2016
+        sed -i '/eval "$(pyenv virtualenv-init -)"/d' "$HOME"/.bashrc
+        # Add initialization lines to .bashrc
+        # shellcheck disable=SC2016
+        cat <<'EOT' | cat - "$HOME"/.bashrc > temp && mv temp "$HOME"/.bashrc
+export PYENV_ROOT="$HOME/.pyenv"
+command -v pyenv >/dev/null || export PATH="$PYENV_ROOT/bin:$PATH"
+eval "$(pyenv init -)"
+eval "$(pyenv virtualenv-init -)"
+EOT
+        # install python
+        source "$HOME"/.bashrc
+        pyenv install 3.12
+    fi
+EOF
+
+#### OpenCV dependencies
+$sudo $pac install -y libgl1
+
+#### Poppler
+# Install poppler
+$sudo $pac install -y poppler-utils
+
+#### OpenOffice / MSOffice doc conversion capabilities
+$sudo $pac install -y libreoffice pandoc
+
+# Install tesseract 5 as well as Russian language
+$sudo $pac install -y software-properties-common
+$sudo add-apt-repository -y ppa:alex-p/tesseract-ocr5
+$sudo $pac install -y tesseract-ocr libtesseract-dev tesseract-ocr-rus
+
+#### libmagic
+$sudo $pac install -y libmagic-dev
+
+#### Put needrestart back the way it was and clean up
+if [[ -d /etc/needrestart/conf.d/ ]]; then
+  $sudo rm -f /etc/needrestart/conf.d/99z_temp_disable.conf
+fi

scripts/sftp-test-helpers/create-and-check-sftp.sh

@@ -0,0 +1,16 @@
+#!/usr/bin/env bash
+
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+
+# Upload nested folder path to sftp server
+function upload() {
+  docker cp "$SCRIPT_DIR"/folder1/ sftp-test:/home/foo/upload/
+}
+
+# Create sftp server
+docker compose version
+docker compose -f "$SCRIPT_DIR"/docker-compose.yaml up --wait
+docker compose -f "$SCRIPT_DIR"/docker-compose.yaml ps
+
+echo "Cluster is live."
+upload

scripts/sftp-test-helpers/docker-compose.yaml

@@ -0,0 +1,12 @@
+services:
+  sftp:
+    image: atmoz/sftp
+    container_name: sftp-test
+    ports:
+      - "47474:22"  # rarely used port in case standard port is used in CI
+    command: foo:bar:::upload
+    healthcheck:
+      test: ["CMD-SHELL", "ssh -o 'BatchMode=true' -o 'StrictHostKeyChecking=no' healthcheck-invalid@localhost 2>&1 | grep -q 'Permission denied'"]
+      interval: 30s
+      timeout: 10s
+      retries: 3

scripts/sftp-test-helpers/folder1/folder2/wiki_movie_plots_small2.csv

@@ -0,0 +1,31 @@
+Release Year,Title,Origin/Ethnicity,Director,Cast,Genre,Wiki Page,Plot
+1901,Kansas Saloon Smashers,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Kansas_Saloon_Smashers,"A bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]"
+1901,Love by the Light of the Moon,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Love_by_the_Light_of_the_Moon,"The moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better."
+1901,The Martyred Presidents,American,Unknown,,unknown,https://en.wikipedia.org/wiki/The_Martyred_Presidents,"The film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination.
+In the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice."
+1901,"Terrible Teddy, the Grizzly King",American,Unknown,,unknown,"https://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_King","Lasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading ""His Photographer"" and ""His Press Agent"" respectively, follow him into the shot; the photographer sets up his camera. ""Teddy"" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. ""Teddy"" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. ""Teddy"" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs."
+1902,Jack and the Beanstalk,American,"George S. Fleming, Edwin S. Porter",,unknown,https://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film),"The earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince."
+1903,Alice in Wonderland,American,Cecil Hepworth,May Clark,unknown,https://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film),"Alice follows a large white rabbit down a ""Rabbit-hole"". She finds a tiny door. When she finds a bottle labeled ""Drink me"", she does, and shrinks, but not enough to pass through the door. She then eats something labeled ""Eat me"" and grows larger. She finds a fan when enables her to shrink enough to get into the ""Garden"" and try to get a ""Dog"" to play with her. She enters the ""White Rabbit's tiny House,"" but suddenly resumes her normal size. In order to get out, she has to use the ""magic fan.""
+She enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. ""The Duchess's Cheshire Cat"" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's ""Mad Tea-Party."" After a while, she leaves.
+The Queen invites Alice to join the ""ROYAL PROCESSION"": a parade of marching playing cards and others headed by the White Rabbit. When Alice ""unintentionally offends the Queen"", the latter summons the ""Executioner"". Alice ""boxes the ears"", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream."
+1903,The Great Train Robbery,American,Edwin S. Porter,,western,https://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film),"The film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left.
+Meanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water.
+There is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail."
+1904,The Suburbanite,American,Wallace McCutcheon,,comedy,https://en.wikipedia.org/wiki/The_Suburbanite,"The film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest."
+1905,The Little Train Robbery,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Little_Train_Robbery,"The opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The ""Bandit Queen,"" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The ""Bandit Queen"" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin.
+The next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the ""valuables,"" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies.
+In the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the ""plunder."" The police, however, have struck the right trail and are in close pursuit. While the ""plunder"" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the ""Bandit Queen."""
+1905,The Night Before Christmas,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film),"Scenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents."
+1906,Dream of a Rarebit Fiend,American,Wallace McCutcheon and Edwin S. Porter,,short,https://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film),"The Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed."
+1906,From Leadville to Aspen: A Hold-Up in the Rockies,American,Francis J. Marion and Wallace McCutcheon,,short action/crime western,https://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_Rockies,The film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart.
+1906,Kathleen Mavourneen,American,Edwin S. Porter,,short film,https://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film),"Irish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration.
+Film historian Charles Musser writes of Porter's adaptation, ""O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.""[1]"
+1907,Daniel Boone,American,Wallace McCutcheon and Ediwin S. Porter,"William Craven, Florence Lawrence",biographical,https://en.wikipedia.org/wiki/Daniel_Boone_(1907_film),"Boone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]"
+1907,How Brown Saw the Baseball Game,American,Unknown,Unknown,comedy,https://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_Game,"Before heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]"
+1907,Laughing Gas,American,Edwin Stanton Porter,"Bertha Regustus, Edward Boulden",comedy,https://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_Film,"The plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets ""catches"" the laughter from her, including a vendor and police officers."
+1908,The Adventures of Dollie,American,D. W. Griffith,"Arthur V. Johnson, Linda Arvidson",drama,https://en.wikipedia.org/wiki/The_Adventures_of_Dollie,"On a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents."
+1908,The Black Viper,American,D. W. Griffith,D. W. Griffith,drama,https://en.wikipedia.org/wiki/The_Black_Viper,"A thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house."
+1908,A Calamitous Elopement,American,D.W. Griffith,"Harry Solter, Linda Arvidson",comedy,https://en.wikipedia.org/wiki/A_Calamitous_Elopement,"A young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings."
+1908,The Call of the Wild,American,D. W. Griffith,Charles Inslee,adventure,https://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film),"A white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as ""The Biograph Girl."""
+1908,A Christmas Carol,American,Unknown,Tom Ricketts,drama,https://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film),"No prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life."
+1908,The Fight for Freedom,American,D. W. Griffith,"Florence Auer, John G. Adolfi",western,https://en.wikipedia.org/wiki/The_Fight_for_Freedom,"The film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town."

scripts/sftp-test-helpers/folder1/wiki_movie_plots_small.csv

@@ -0,0 +1,31 @@
+Release Year,Title,Origin/Ethnicity,Director,Cast,Genre,Wiki Page,Plot
+1901,Kansas Saloon Smashers,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Kansas_Saloon_Smashers,"A bartender is working at a saloon, serving drinks to customers. After he fills a stereotypically Irish man's bucket with beer, Carrie Nation and her followers burst inside. They assault the Irish man, pulling his hat over his eyes and then dumping the beer over his head. The group then begin wrecking the bar, smashing the fixtures, mirrors, and breaking the cash register. The bartender then sprays seltzer water in Nation's face before a group of policemen appear and order everybody to leave.[1]"
+1901,Love by the Light of the Moon,American,Unknown,,unknown,https://en.wikipedia.org/wiki/Love_by_the_Light_of_the_Moon,"The moon, painted with a smiling face hangs over a park at night. A young couple walking past a fence learn on a railing and look up. The moon smiles. They embrace, and the moon's smile gets bigger. They then sit down on a bench by a tree. The moon's view is blocked, causing him to frown. In the last scene, the man fans the woman with his hat because the moon has left the sky and is perched over her shoulder to see everything better."
+1901,The Martyred Presidents,American,Unknown,,unknown,https://en.wikipedia.org/wiki/The_Martyred_Presidents,"The film, just over a minute long, is composed of two shots. In the first, a girl sits at the base of an altar or tomb, her face hidden from the camera. At the center of the altar, a viewing portal displays the portraits of three U.S. Presidents—Abraham Lincoln, James A. Garfield, and William McKinley—each victims of assassination.
+In the second shot, which runs just over eight seconds long, an assassin kneels feet of Lady Justice."
+1901,"Terrible Teddy, the Grizzly King",American,Unknown,,unknown,"https://en.wikipedia.org/wiki/Terrible_Teddy,_the_Grizzly_King","Lasting just 61 seconds and consisting of two shots, the first shot is set in a wood during winter. The actor representing then vice-president Theodore Roosevelt enthusiastically hurries down a hillside towards a tree in the foreground. He falls once, but rights himself and cocks his rifle. Two other men, bearing signs reading ""His Photographer"" and ""His Press Agent"" respectively, follow him into the shot; the photographer sets up his camera. ""Teddy"" aims his rifle upward at the tree and fells what appears to be a common house cat, which he then proceeds to stab. ""Teddy"" holds his prize aloft, and the press agent takes notes. The second shot is taken in a slightly different part of the wood, on a path. ""Teddy"" rides the path on his horse towards the camera and out to the left of the shot, followed closely by the press agent and photographer, still dutifully holding their signs."
+1902,Jack and the Beanstalk,American,"George S. Fleming, Edwin S. Porter",,unknown,https://en.wikipedia.org/wiki/Jack_and_the_Beanstalk_(1902_film),"The earliest known adaptation of the classic fairytale, this films shows Jack trading his cow for the beans, his mother forcing him to drop them in the front yard, and beig forced upstairs. As he sleeps, Jack is visited by a fairy who shows him glimpses of what will await him when he ascends the bean stalk. In this version, Jack is the son of a deposed king. When Jack wakes up, he finds the beanstalk has grown and he climbs to the top where he enters the giant's home. The giant finds Jack, who narrowly escapes. The giant chases Jack down the bean stalk, but Jack is able to cut it down before the giant can get to safety. He falls and is killed as Jack celebrates. The fairy then reveals that Jack may return home as a prince."
+1903,Alice in Wonderland,American,Cecil Hepworth,May Clark,unknown,https://en.wikipedia.org/wiki/Alice_in_Wonderland_(1903_film),"Alice follows a large white rabbit down a ""Rabbit-hole"". She finds a tiny door. When she finds a bottle labeled ""Drink me"", she does, and shrinks, but not enough to pass through the door. She then eats something labeled ""Eat me"" and grows larger. She finds a fan when enables her to shrink enough to get into the ""Garden"" and try to get a ""Dog"" to play with her. She enters the ""White Rabbit's tiny House,"" but suddenly resumes her normal size. In order to get out, she has to use the ""magic fan.""
+She enters a kitchen, in which there is a cook and a woman holding a baby. She persuades the woman to give her the child and takes the infant outside after the cook starts throwing things around. The baby then turns into a pig and squirms out of her grip. ""The Duchess's Cheshire Cat"" appears and disappears a couple of times to Alice and directs her to the Mad Hatter's ""Mad Tea-Party."" After a while, she leaves.
+The Queen invites Alice to join the ""ROYAL PROCESSION"": a parade of marching playing cards and others headed by the White Rabbit. When Alice ""unintentionally offends the Queen"", the latter summons the ""Executioner"". Alice ""boxes the ears"", then flees when all the playing cards come for her. Then she wakes up and realizes it was all a dream."
+1903,The Great Train Robbery,American,Edwin S. Porter,,western,https://en.wikipedia.org/wiki/The_Great_Train_Robbery_(1903_film),"The film opens with two bandits breaking into a railroad telegraph office, where they force the operator at gunpoint to have a train stopped and to transmit orders for the engineer to fill the locomotive's tender at the station's water tank. They then knock the operator out and tie him up. As the train stops it is boarded by the bandits‍—‌now four. Two bandits enter an express car, kill a messenger and open a box of valuables with dynamite; the others kill the fireman and force the engineer to halt the train and disconnect the locomotive. The bandits then force the passengers off the train and rifle them for their belongings. One passenger tries to escape but is instantly shot down. Carrying their loot, the bandits escape in the locomotive, later stopping in a valley where their horses had been left.
+Meanwhile, back in the telegraph office, the bound operator awakens, but he collapses again. His daughter arrives bringing him his meal and cuts him free, and restores him to consciousness by dousing him with water.
+There is some comic relief at a dance hall, where an Eastern stranger is forced to dance while the locals fire at his feet. The door suddenly opens and the telegraph operator rushes in to tell them of the robbery. The men quickly form a posse, which overtakes the bandits, and in a final shootout kills them all and recovers the stolen mail."
+1904,The Suburbanite,American,Wallace McCutcheon,,comedy,https://en.wikipedia.org/wiki/The_Suburbanite,"The film is about a family who move to the suburbs, hoping for a quiet life. Things start to go wrong, and the wife gets violent and starts throwing crockery, leading to her arrest."
+1905,The Little Train Robbery,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Little_Train_Robbery,"The opening scene shows the interior of the robbers' den. The walls are decorated with the portraits of notorious criminals and pictures illustrating the exploits of famous bandits. Some of the gang are lounging about, while others are reading novels and illustrated papers. Although of youthful appearance, each is dressed like a typical Western desperado. The ""Bandit Queen,"" leading a blindfolded new recruit, now enters the room. He is led to the center of the room, raises his right hand and is solemnly sworn in. When the bandage is removed from his eyes he finds himself looking into the muzzles of a dozen or more 45's. The gang then congratulates the new member and heartily shake his hand. The ""Bandit Queen"" who is evidently the leader of the gang, now calls for volunteers to hold up a train. All respond, but she picks out seven for the job who immediately leave the cabin.
+The next scene shows the gang breaking into a barn. They steal ponies and ride away. Upon reaching the place agreed upon they picket their ponies and leaving them in charge of a trusted member proceed to a wild mountain spot in a bend of the railroad, where the road runs over a steep embankment. The spot is an ideal one for holding up a train. Cross ties are now placed on the railroad track and the gang hide in some bushes close by and wait for the train. The train soon approaches and is brought to a stop. The engineer leaves his engine and proceeds to remove the obstruction on the track. While he is bending over one of the gang sneaks up behind them and hits him on the head with an axe, and knocks him senseless down the embankment, while the gang surround the train and hold up the passengers. After securing all the ""valuables,"" consisting principally of candy and dolls, the robbers uncouple the engine and one car and make their escape just in time to avoid a posse of police who appear on the scene. Further up the road they abandon the engine and car, take to the woods and soon reach their ponies.
+In the meantime the police have learned the particulars of the hold-up from the frightened passengers and have started up the railroad tracks after the fleeing robbers. The robbers are next seen riding up the bed of a shallow stream and finally reach their den, where the remainder of the gang have been waiting for them. Believing they have successfully eluded their pursuers, they proceed to divide the ""plunder."" The police, however, have struck the right trail and are in close pursuit. While the ""plunder"" is being divided a sentry gives the alarm and the entire gang, abandoning everything, rush from the cabin barely in time to escape capture. The police make a hurried search and again start in pursuit. The robbers are so hard pressed that they are unable to reach their ponies, and are obliged to take chances on foot. The police now get in sight of the fleeing robbers and a lively chase follows through tall weeds, over a bridge and up a steep hill. Reaching a pond the police are close on their heels. The foremost robbers jump in clothes and all and strike out for the opposite bank. Two hesitate and are captured. Boats are secured and after an exciting tussle the entire gang is rounded up. In the mix up one of the police is dragged overboard. The final scene shows the entire gang of bedraggled and crestfallen robbers tied together with a rope and being led away by the police. Two of the police are loaded down with revolvers, knives and cartridge belts, and resemble walking aresenals. As a fitting climax a confederate steals out of the woods, cuts the rope and gallantly rescues the ""Bandit Queen."""
+1905,The Night Before Christmas,American,Edwin Stanton Porter,,unknown,https://en.wikipedia.org/wiki/The_Night_Before_Christmas_(1905_film),"Scenes are introduced using lines of the poem.[2] Santa Claus, played by Harry Eytinge, is shown feeding real reindeer[4] and finishes his work in the workshop. Meanwhile, the children of a city household hang their stockings and go to bed, but unable to sleep they engage in a pillow fight. Santa Claus leaves his home on a sleigh with his reindeer. He enters the children's house through the chimney, and leaves the presents. The children come down the stairs and enjoy their presents."
+1906,Dream of a Rarebit Fiend,American,Wallace McCutcheon and Edwin S. Porter,,short,https://en.wikipedia.org/wiki/Dream_of_a_Rarebit_Fiend_(1906_film),"The Rarebit Fiend gorges on Welsh rarebit at a restaurant. When he leaves, he begins to get dizzy as he starts to hallucinate. He desperately tries to hang onto a lamppost as the world spins all around him. A man helps him get home. He falls into bed and begins having more hallucinatory dreams. During a dream sequence, the furniture begins moving around the room. Imps emerge from a floating Welsh rarebit container and begin poking his head as he sleeps. His bed then begins dancing and spinning wildly around the room before flying out the window with the Fiend in it. The bed floats across the city as the Fiend floats up and off the bed. He hangs off the back and eventually gets caught on a weathervane atop a steeple. His bedclothes tear and he falls from the sky, crashing through his bedroom ceiling. The Fiend awakens from the dream after falling out of his bed."
+1906,From Leadville to Aspen: A Hold-Up in the Rockies,American,Francis J. Marion and Wallace McCutcheon,,short action/crime western,https://en.wikipedia.org/wiki/From_Leadville_to_Aspen:_A_Hold-Up_in_the_Rockies,The film features a train traveling through the Rockies and a hold up created by two thugs placing logs on the line. They systematically rob the wealthy occupants at gunpoint and then make their getaway along the tracks and later by a hi-jacked horse and cart.
+1906,Kathleen Mavourneen,American,Edwin S. Porter,,short film,https://en.wikipedia.org/wiki/Kathleen_Mavourneen_(1906_film),"Irish villager Kathleen is a tenant of Captain Clearfield, who controls local judges and criminals. Her father owes Clearfield a large debt. Terence O'More saves the village from Clearfield, causing a large celebration.
+Film historian Charles Musser writes of Porter's adaptation, ""O'More not only rescues Kathleen from the villain but, through marriage, renews the family for another generation.""[1]"
+1907,Daniel Boone,American,Wallace McCutcheon and Ediwin S. Porter,"William Craven, Florence Lawrence",biographical,https://en.wikipedia.org/wiki/Daniel_Boone_(1907_film),"Boone's daughter befriends an Indian maiden as Boone and his companion start out on a hunting expedition. While he is away, Boone's cabin is attacked by the Indians, who set it on fire and abduct Boone's daughter. Boone returns, swears vengeance, then heads out on the trail to the Indian camp. His daughter escapes but is chased. The Indians encounter Boone, which sets off a huge fight on the edge of a cliff. A burning arrow gets shot into the Indian camp. Boone gets tied to the stake and tortured. The burning arrow sets the Indian camp on fire, causing panic. Boone is rescued by his horse, and Boone has a knife fight in which he kills the Indian chief.[2]"
+1907,How Brown Saw the Baseball Game,American,Unknown,Unknown,comedy,https://en.wikipedia.org/wiki/How_Brown_Saw_the_Baseball_Game,"Before heading out to a baseball game at a nearby ballpark, sports fan Mr. Brown drinks several highball cocktails. He arrives at the ballpark to watch the game, but has become so inebriated that the game appears to him in reverse, with the players running the bases backwards and the baseball flying back into the pitcher's hand. After the game is over, Mr. Brown is escorted home by one of his friends. When they arrive at Brown's house, they encounter his wife who becomes furious with the friend and proceeds to physically assault him, believing he is responsible for her husband's severe intoxication.[1]"
+1907,Laughing Gas,American,Edwin Stanton Porter,"Bertha Regustus, Edward Boulden",comedy,https://en.wikipedia.org/wiki/Laughing_Gas_(film)#1907_Film,"The plot is that of a black woman going to the dentist for a toothache and being given laughing gas. On her way walking home, and in other situations, she can't stop laughing, and everyone she meets ""catches"" the laughter from her, including a vendor and police officers."
+1908,The Adventures of Dollie,American,D. W. Griffith,"Arthur V. Johnson, Linda Arvidson",drama,https://en.wikipedia.org/wiki/The_Adventures_of_Dollie,"On a beautiful summer day a father and mother take their daughter Dollie on an outing to the river. The mother refuses to buy a gypsy's wares. The gypsy tries to rob the mother, but the father drives him off. The gypsy returns to the camp and devises a plan. They return and kidnap Dollie while her parents are distracted. A rescue crew is organized, but the gypsy takes Dollie to his camp. They gag Dollie and hide her in a barrel before the rescue party gets to the camp. Once they leave the gypsies and escapes in their wagon. As the wagon crosses the river, the barrel falls into the water. Still sealed in the barrel, Dollie is swept downstream in dangerous currents. A boy who is fishing in the river finds the barrel, and Dollie is reunited safely with her parents."
+1908,The Black Viper,American,D. W. Griffith,D. W. Griffith,drama,https://en.wikipedia.org/wiki/The_Black_Viper,"A thug accosts a girl as she leaves her workplace but a man rescues her. The thug vows revenge and, with the help of two friends, attacks the girl and her rescuer again as they're going for a walk. This time they succeed in kidnapping the rescuer. He is bound and gagged and taken away in a cart. The girl runs home and gets help from several neighbors. They track the ruffians down to a cabin in the mountains where the gang has trapped their victim and set the cabin on fire. A thug and Rescuer fight on the roof of the house."
+1908,A Calamitous Elopement,American,D.W. Griffith,"Harry Solter, Linda Arvidson",comedy,https://en.wikipedia.org/wiki/A_Calamitous_Elopement,"A young couple decides to elope after being caught in the midst of a romantic moment by the woman's angry father. They make plans to leave, but a thief discovers their plans and hides in their trunk and waits for the right moment to steal their belongings."
+1908,The Call of the Wild,American,D. W. Griffith,Charles Inslee,adventure,https://en.wikipedia.org/wiki/The_Call_of_the_Wild_(1908_film),"A white girl (Florence Lawrence) rejects a proposal from an Indian brave (Charles Inslee) in this early one-reel Western melodrama. Despite the rejection, the Indian still comes to the girl's defense when she is abducted by his warring tribe. In her first year in films, Florence Lawrence was already the most popular among the Biograph Company's anonymous stock company players. By 1909, she was known the world over as ""The Biograph Girl."""
+1908,A Christmas Carol,American,Unknown,Tom Ricketts,drama,https://en.wikipedia.org/wiki/A_Christmas_Carol_(1908_film),"No prints of the first American film adaptation of A Christmas Carol are known to exist,[1] but The Moving Picture World magazine provided a scene-by-scene description before the film's release.[2] Scrooge goes into his office and begins working. His nephew, along with three women who wish for Scrooge to donate enter. However, Scrooge dismisses them. On the night of Christmas Eve, his long-dead partner Jacob Marley comes as a ghost, warning him of a horrible fate if he does not change his ways. Scrooge meets three spirits that show Scrooge the real meaning of Christmas, along with his grave, the result of his parsimonious ways. The next morning, he wakes and realizes the error of his ways. Scrooge was then euphoric and generous for the rest of his life."
+1908,The Fight for Freedom,American,D. W. Griffith,"Florence Auer, John G. Adolfi",western,https://en.wikipedia.org/wiki/The_Fight_for_Freedom,"The film opens in a town on the Mexican border. A poker game is going on in the local saloon. One of the players cheats and is shot dead by another of the players, a Mexican named Pedro. In the uproar that follows Pedro is wounded as he escapes from the saloon. The sheriff is called, who tracks Pedro to his home but Pedro kills the sherriff too. While Pedro hides, his wife Juanita, is arrested on suspicion of murdering the sheriff. Pedro rescues her from the town jail and the two head for the Mexican border. Caught by the posse before they reach the border, Juanita is killed and the film ends with Pedro being arrested and taken back to town."

scripts/shellcheck.sh

@@ -0,0 +1,3 @@
+#!/usr/bin/env bash
+
+find . -name "*.sh" -exec shellcheck {} +

scripts/singlestore-test-helpers/docker-compose.yml

@@ -0,0 +1,21 @@
+services:
+  singlestore:
+    container_name: "singlestore"
+    image: ghcr.io/singlestore-labs/singlestoredb-dev:latest
+    platform: linux/amd64
+    ports:
+      - 3306:3306
+      - 8080:8080
+      - 9000:9000
+    environment:
+      - ROOT_PASSWORD=password
+    volumes:
+      - ./schema.sql:/init.sql
+
+  # Allow docker compose up --wait to exit only when singlestore is healthy
+  wait:
+    image: hello-world:latest
+    container_name: singlestore-waiter
+    depends_on:
+      singlestore:
+        condition: service_healthy

scripts/singlestore-test-helpers/schema.sql

@@ -0,0 +1,48 @@
+CREATE DATABASE ingest_test;
+USE ingest_test;
+
+CREATE TABLE elements (
+    id INT PRIMARY KEY NOT NULL AUTO_INCREMENT,
+    element_id TEXT,
+    text TEXT,
+    embeddings Vector(384),
+    type TEXT,
+    url TEXT,
+    version TEXT,
+    data_source_date_created TIMESTAMP,
+    data_source_date_modified TIMESTAMP,
+    data_source_date_processed TIMESTAMP,
+    data_source_permissions_data TEXT,
+    data_source_url TEXT,
+    data_source_version TEXT,
+    data_source_record_locator JSON,
+    category_depth INTEGER,
+    parent_id TEXT,
+    attached_filename TEXT,
+    filetype TEXT,
+    last_modified TIMESTAMP,
+    file_directory TEXT,
+    filename TEXT,
+    languages TEXT,
+    page_number TEXT,
+    links TEXT,
+    page_name TEXT,
+    link_urls TEXT,
+    link_texts TEXT,
+    sent_from TEXT,
+    sent_to TEXT,
+    subject TEXT,
+    section TEXT,
+    header_footer_type TEXT,
+    emphasized_text_contents TEXT,
+    emphasized_text_tags TEXT,
+    text_as_html TEXT,
+    detection_class_prob DECIMAL,
+    is_continuation BOOLEAN,
+    orig_elements TEXT,
+    coordinates_points TEXT,
+    coordinates_system TEXT,
+    coordinates_layout_width DECIMAL,
+    coordinates_layout_height DECIMAL
+);
+

scripts/singlestore-test-helpers/test_outputs.py

@@ -0,0 +1,56 @@
+#!/usr/bin/env python3
+
+import click
+import singlestoredb as s2
+from singlestoredb.connection import Connection
+
+
+def get_connection(
+    host: str = None, port: int = None, database: str = None, user: str = None, password: str = None
+) -> Connection:
+    conn = s2.connect(
+        host=host,
+        port=port,
+        database=database,
+        user=user,
+        password=password,
+    )
+    return conn
+
+
+def validate(table_name: str, conn: Connection, num_elements: int):
+    with conn.cursor() as cur:
+        stmt = f"select * from {table_name}"
+        count = cur.execute(stmt)
+        assert count == num_elements, (
+            f"found count ({count}) doesn't match expected value: {num_elements}"
+        )
+    print("validation successful")
+
+
+@click.command()
+@click.option("--host", type=str, default="localhost", show_default=True)
+@click.option("--port", type=int, default=3306, show_default=True)
+@click.option("--user", type=str, default="root", show_default=True)
+@click.option("--password", type=str, default="password")
+@click.option("--database", type=str, required=True)
+@click.option("--table-name", type=str, required=True)
+@click.option(
+    "--num-elements", type=int, required=True, help="The expected number of elements to exist"
+)
+def run_validation(
+    host: str,
+    port: int,
+    user: str,
+    database: str,
+    password: str,
+    table_name: str,
+    num_elements: int,
+):
+    print(f"Validating that table {table_name} in database {database} has {num_elements} entries")
+    conn = get_connection(host=host, port=port, database=database, user=user, password=password)
+    validate(table_name=table_name, conn=conn, num_elements=num_elements)
+
+
+if __name__ == "__main__":
+    run_validation()

scripts/sql-test-helpers/create-pgvector-schema.sql

@@ -0,0 +1,43 @@
+CREATE EXTENSION vector;
+
+CREATE TABLE elements (
+    id UUID PRIMARY KEY,
+    element_id VARCHAR,
+    text TEXT,
+    embeddings vector(384),
+    type VARCHAR,
+    system VARCHAR,
+    layout_width DECIMAL,
+    layout_height DECIMAL,
+    points TEXT,
+    url TEXT,
+    version VARCHAR,
+    date_created TIMESTAMPTZ,
+    date_modified TIMESTAMPTZ,
+    date_processed TIMESTAMPTZ,
+    permissions_data TEXT,
+    record_locator TEXT,
+    category_depth INTEGER,
+    parent_id VARCHAR,
+    attached_filename VARCHAR,
+    filetype VARCHAR,
+    last_modified TIMESTAMPTZ,
+    file_directory VARCHAR,
+    filename VARCHAR,
+    languages VARCHAR [],
+    page_number VARCHAR,
+    links TEXT,
+    page_name VARCHAR,
+    link_urls VARCHAR [],
+    link_texts VARCHAR [],
+    sent_from VARCHAR [],
+    sent_to VARCHAR [],
+    subject VARCHAR,
+    section VARCHAR,
+    header_footer_type VARCHAR,
+    emphasized_text_contents VARCHAR [],
+    emphasized_text_tags VARCHAR [],
+    text_as_html TEXT,
+    detection_class_prob DECIMAL
+);
+

scripts/sql-test-helpers/create-sql-instance.sh

@@ -0,0 +1,20 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+DATABASE_NAME=$1
+DATABASE_FILE_PATH=$2
+
+# Create the SQL instance
+if [[ "$DATABASE_NAME" != "sqlite" ]]; then
+  docker compose version
+  docker compose -f "$SCRIPT_DIR"/docker-compose-"$DATABASE_NAME".yaml up --wait
+  docker compose -f "$SCRIPT_DIR"/docker-compose-"$DATABASE_NAME".yaml ps
+else
+  touch "$DATABASE_FILE_PATH"
+
+  python "$SCRIPT_DIR"/create-sqlite-schema.py "$DATABASE_FILE_PATH"
+fi
+
+echo "$DATABASE_NAME instance is live."

scripts/sql-test-helpers/create-sqlite-schema.py

@@ -0,0 +1,14 @@
+import sqlite3
+import sys
+from pathlib import Path
+
+if __name__ == "__main__":
+    connection = sqlite3.connect(database=sys.argv[1])
+
+    query = None
+    script_path = (Path(__file__).parent / Path("create-sqlite-schema.sql")).resolve()
+    with open(script_path) as f:
+        query = f.read()
+    cursor = connection.cursor()
+    cursor.executescript(query)
+    connection.close()

scripts/sql-test-helpers/create-sqlite-schema.sql

@@ -0,0 +1,40 @@
+CREATE TABLE elements (
+    id TEXT PRIMARY KEY,
+    element_id TEXT,
+    text TEXT,
+    embeddings TEXT,
+    type TEXT,
+    system TEXT,
+    layout_width REAL,
+    layout_height REAL,
+    points TEXT,
+    url TEXT,
+    version TEXT,
+    date_created TEXT,
+    date_modified TEXT,
+    date_processed TEXT,
+    permissions_data TEXT,
+    record_locator TEXT,
+    category_depth INTEGER,
+    parent_id TEXT,
+    attached_filename TEXT,
+    filetype TEXT,
+    last_modified TEXT,
+    file_directory TEXT,
+    filename TEXT,
+    languages TEXT,
+    page_number TEXT,
+    links TEXT,
+    page_name TEXT,
+    link_urls TEXT,
+    link_texts TEXT,
+    sent_from TEXT,
+    sent_to TEXT,
+    subject TEXT,
+    section TEXT,
+    header_footer_type TEXT,
+    emphasized_text_contents TEXT,
+    emphasized_text_tags TEXT,
+    text_as_html TEXT,
+    detection_class_prob DECIMAL
+);

scripts/sql-test-helpers/docker-compose-pgvector.yaml

@@ -0,0 +1,13 @@
+services:
+  pgvector:
+    image: ankane/pgvector
+    restart: always
+    container_name: pgvector_dest
+    ports:
+     - 5433:5432
+    environment:
+      POSTGRES_DB: elements
+      POSTGRES_USER: unstructured
+      POSTGRES_PASSWORD: test
+    volumes:
+      - ./create-pgvector-schema.sql:/docker-entrypoint-initdb.d/init.sql

scripts/sync_fork.sh

@@ -0,0 +1,29 @@
+#!/bin/bash
+
+# Simple script to recreate a fork branch as a new branch in the current repository
+# Usage: ./sync_fork.sh <fork_url> <fork_branch>
+
+set -e
+
+if [ $# -ne 2 ]; then
+  echo "Usage: $0 <fork_url> <fork_branch>"
+  echo "Example: $0 https://github.com/user/fork.git feature-branch"
+  exit 1
+fi
+
+FORK_URL="$1"
+FORK_BRANCH="$2"
+
+echo "Adding fork as remote..."
+git remote add fork "$FORK_URL" 2>/dev/null || git remote set-url fork "$FORK_URL"
+
+echo "Fetching fork..."
+git fetch fork
+
+echo "Creating new branch '$FORK_BRANCH' with fork's changes..."
+git checkout -b "$FORK_BRANCH" "fork/$FORK_BRANCH"
+
+echo "Removing fork remote..."
+git remote remove fork
+
+echo "Done! You're now on branch '$FORK_BRANCH' with the fork's changes. Fork remote has been removed."

scripts/user/process-pdf-parallel-through-api.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+# Usage: ./process-pdf-parallel-through-api.sh filename.pdf
+
+set -eu -o pipefail
+
+if [ $# -ne 1 ]; then
+  echo "Processes a single PDF through the Unstructured API by breaking it into smaller splits that are processed concurrently."
+  echo
+  echo "Usage: $0 <pdf_filename>"
+  echo "Please provide a PDF filename as the first argument."
+  echo
+  echo "Optionally, set the following env vars: "
+  echo
+  echo "* STRATEGY (default hi_res)"
+  echo "* BATCH_SIZE (default 30) as the number of parts (AKA splits) to process in parallel"
+  echo "* PDF_SPLIT_PAGE_SIZE (default 10) as the number of pages per split"
+  echo
+  echo "BATCH_SIZE=20 PDF_SPLIT_PAGE_SIZE=6 STRATEGY=hi_res ./process-pdf-parallel-through-api.sh example-docs/pdf/layout-parser-paper.pdf"
+  exit 1
+fi
+
+ALLOWED_STRATEGIES=("hi_res" "fast" "auto")
+
+# Validate STRATEGY environment variable if it's set
+if [ -n "${STRATEGY:-}" ] && [[ ! " ${ALLOWED_STRATEGIES[*]} " =~ ${STRATEGY} ]]; then
+  echo "Error: STRATEGY must be one of ${ALLOWED_STRATEGIES[*]}" >&2
+  exit 1
+fi
+
+# Check if UNST_API_KEY is set
+if [ -z "${UNST_API_KEY}" ]; then
+  echo "Error: UNST_API_KEY is not set or is empty" >&2
+  exit 1
+fi
+
+PDF_FILE="$1"
+DEFAULT_SPLIT_SIZE=10
+SPLIT_SIZE=${PDF_SPLIT_PAGE_SIZE:-$DEFAULT_SPLIT_SIZE}
+PDF_NAME=$(basename "$PDF_FILE" .pdf)
+DEFAULT_DIR="$HOME/tmp/pdf-splits"
+PDF_SPLITS_DIR="${PDF_SPLITS_DIR:-$DEFAULT_DIR}"
+MD5_SUM=$(md5sum "$PDF_FILE" | awk '{ print $1 }')
+PDF_DIR="$PDF_SPLITS_DIR/$PDF_NAME-${MD5_SUM}_split-${SPLIT_SIZE}"
+PDF_OUTPUT_DIR="$PDF_SPLITS_DIR/${PDF_NAME}-output-${MD5_SUM}_split-${SPLIT_SIZE}_strat-${STRATEGY}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Check if PDF parts directory exists
+if [ ! -d "$PDF_DIR" ]; then
+  "$SCRIPT_DIR/split-pdf.sh" "$PDF_FILE"
+fi
+
+# Create output directory if it does not exist
+mkdir -p "$PDF_OUTPUT_DIR"
+
+incomplete=0 # Flag to track incomplete processing
+
+# Function to process a single PDF part file
+process_file_part() {
+  local file="$1"
+  local STARTING_PAGE_NUMBER="$2"
+  local OUTPUT_JSON="$3"
+
+  if [ -f "$OUTPUT_JSON" ]; then
+    echo "Skipping processing for $OUTPUT_JSON as it already exists."
+    return
+  fi
+
+  curl -q -X POST https://api.unstructuredapp.io/general/v0/general \
+    -H "unstructured-api-key: $UNST_API_KEY" \
+    -H 'accept: application/json' \
+    -H 'Content-Type: multipart/form-data' \
+    -F strategy="${STRATEGY:-hi_res}" \
+    -F 'skip_infer_table_types="[]"' \
+    -F starting_page_number="$STARTING_PAGE_NUMBER" \
+    -F files=@"$file;filename=$PDF_FILE" \
+    -o "$OUTPUT_JSON"
+
+  # Verify JSON content
+  if ! jq -e 'if type=="array" then all(.[]; type=="object" or length==0) else empty end' "$OUTPUT_JSON" >/dev/null; then
+    echo "Invalid JSON structure in $OUTPUT_JSON (contents below), deleting file."
+    cat "$OUTPUT_JSON"
+    rm "$OUTPUT_JSON"
+    incomplete=1
+  else
+    echo "Valid JSON output created: $OUTPUT_JSON"
+  fi
+}
+
+# Function to process a batch of files
+process_batch() {
+  for file in "$@"; do
+    local START_PAGE
+    START_PAGE=$(echo "$file" | sed -n 's/.*_pages_\([0-9]*\)_to_[0-9]*.pdf/\1/p')
+    local END_PAGE=
+    END_PAGE=$(echo "$file" | sed -n 's/.*_pages_[0-9]*_to_\([0-9]*\).pdf/\1/p')
+    local OUTPUT_JSON="$PDF_OUTPUT_DIR/${PDF_NAME}_pages_${START_PAGE}_to_${END_PAGE}.json"
+    process_file_part "$file" "$START_PAGE" "$OUTPUT_JSON" &
+  done
+  wait
+}
+
+# Read PDF parts into an array
+mapfile -t pdf_parts < <(find "$PDF_DIR" -name '*.pdf' -print)
+
+# Process PDF parts in batches of 30, by default
+batch_size=${BATCH_SIZE:-30}
+for ((i = 0; i < ${#pdf_parts[@]}; i += batch_size)); do
+  process_batch "${pdf_parts[@]:i:batch_size}"
+done
+
+# Determine the output filename based on whether processing was incomplete
+if [ "$incomplete" -eq 1 ]; then
+  combined_output_filename="${PDF_NAME}_incomplete_combined.json"
+  echo "WARNING! not all json parts were successfully processed. you may rerun this script"
+  echo "to attempt reprocessing those (failed to process) parts."
+else
+  combined_output_filename="${PDF_NAME}_combined.json"
+fi
+
+# Combine JSON outputs in numerical order
+find "$PDF_OUTPUT_DIR" -name '*.json' -print0 | sort -zV | xargs -0 jq -s 'add' >"$PDF_OUTPUT_DIR/$combined_output_filename"
+
+echo "Processing complete. Combined JSON saved to $PDF_OUTPUT_DIR/$combined_output_filename"

scripts/user/split-pdf.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Usage: ./split_pdf.sh filename.pdf
+
+set -e
+
+PDF_FILE="$1"
+DEFAULT_SPLIT_SIZE=5
+SPLIT_SIZE=${PDF_SPLIT_PAGE_SIZE:-$DEFAULT_SPLIT_SIZE}
+
+# Validate that SPLIT_SIZE is an integer
+if ! [[ "$SPLIT_SIZE" =~ ^[0-9]+$ ]]; then
+  echo "Error: PDF_SPLIT_PAGE_SIZE must be an integer."
+  exit 1
+fi
+
+DEFAULT_DIR="$HOME/tmp/pdf-splits"
+PDF_SPLITS_DIR="${PDF_SPLITS_DIR:-$DEFAULT_DIR}"
+PDF_NAME=$(basename "$PDF_FILE" .pdf)
+MD5_SUM=$(md5sum "$PDF_FILE" | awk '{ print $1 }')
+PDF_DIR="$PDF_SPLITS_DIR/$PDF_NAME-${MD5_SUM}_split-${SPLIT_SIZE}"
+
+# Create directory if it does not exist
+mkdir -p "$PDF_DIR"
+
+# Total number of pages
+TOTAL_PAGES=$(qpdf --show-npages "$PDF_FILE")
+
+# Split PDF into $SPLIT_SIZE-page chunks
+START_PAGE=1
+while [ "$START_PAGE" -le "$TOTAL_PAGES" ]; do
+  END_PAGE=$((START_PAGE + SPLIT_SIZE - 1))
+  if [ "$END_PAGE" -gt "$TOTAL_PAGES" ]; then
+    END_PAGE=$TOTAL_PAGES
+  fi
+  OUTPUT_FILE="$PDF_DIR/${PDF_NAME}_pages_${START_PAGE}_to_${END_PAGE}.pdf"
+  qpdf "$PDF_FILE" --pages . "$START_PAGE"-"$END_PAGE" -- "$OUTPUT_FILE"
+  echo "Created $OUTPUT_FILE"
+  START_PAGE=$((END_PAGE + 1))
+done
+
+echo "All parts have been saved to $PDF_DIR"

scripts/user/u-tables-inspect.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# TODO:
+# * include the page number in output html filename, maybe all tables per page
+#   in one html page.
+
+set -e
+
+USAGE_MESSAGE="Usage: $0 <file>
+
+Requires an unstructured output .json file as the only argument.
+
+Each table in the file is saved as an individual html file and
+opened in Safari (if running on mac), providing a quick and easy
+way to see the structure and content of each Table element.
+"
+
+# Check for the presence of an argument
+if [ "$#" -ne 1 ]; then
+  echo "$USAGE_MESSAGE"
+  exit 1
+fi
+
+# The JSON file to be processed is the first argument
+JSON_FILE="$1"
+
+# Extract the basename without the extension
+BASE_NAME=$(basename "$JSON_FILE" .json)
+
+# Directory where the files will be saved
+TMP_TABLES_OUTPUTS_DIR="$HOME/tmp/tables-out"
+
+# Check if the directory exists, if not create it
+if [ ! -d "$TMP_TABLES_OUTPUTS_DIR" ]; then
+  mkdir -p "$TMP_TABLES_OUTPUTS_DIR"
+fi
+
+# Counter for the table files
+COUNTER=1
+
+# Parsing the JSON and creating HTML files
+jq -c '.[] | select(.type == "Table") | .metadata.text_as_html' "$JSON_FILE" | while read -r HTML_CONTENT; do
+  # Remove leading and trailing quotes from the JSON string
+  HTML_CONTENT=${HTML_CONTENT#\"}
+  HTML_CONTENT=${HTML_CONTENT%\"}
+  # add a border and padding to clearly see cell definition
+  # shellcheck disable=SC2001
+  HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/<table /<table border="1" cellpadding="10" /')
+  # shellcheck disable=SC2001
+  HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/<table>/<table border="1" cellpadding="10">/')
+  # add newlines for readability in the html
+  # shellcheck disable=SC2001
+  HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/>\s*</>\n</g')
+
+  # Create filename based on the basename and counter
+  HTML_FILENAME="$TMP_TABLES_OUTPUTS_DIR/${BASE_NAME}-${COUNTER}.html"
+
+  # Increment the counter
+  COUNTER=$((COUNTER + 1))
+
+  # Save the HTML content to a file
+  echo "$HTML_CONTENT" >"$HTML_FILENAME"
+
+  if [ "$(uname)" == "Darwin" ]; then
+    # Open the file in a new browser window
+    open -a "Safari" "$HTML_FILENAME" &
+  else
+    echo "$HTML_FILENAME"
+  fi
+done

scripts/user/unstructured-get-json.sh

@@ -0,0 +1,367 @@
+#!/usr/bin/env bash
+
+# TODO's
+# * ability to set file type so that is not inferred by the unstructured api service
+#     e.g. "-F 'files=@foo.pdf;type=application/pdf'
+#
+
+set -e
+
+# shellcheck disable=SC2016
+USAGE_MESSAGE="Usage: $0 [options] <file>"'
+
+Options:
+  --api-key KEY   Specify the API key for authentication. Set the env var $UNST_API_KEY to skip providing this option.
+  --freemium      Use the free API rather paid API
+  --hi-res        hi_res strategy: Enable high-resolution processing, with layout segmentation and OCR
+  --fast          fast strategy: No OCR, just extract embedded text
+  --ocr-only      ocr_only strategy: Perform OCR (Optical Character Recognition) only. No layout segmentation.
+  --vlm           vlm strategy: Use Vision Language Model for processing
+  --vlm-provider  Specify the VLM model provider
+                  (see: https://docs.unstructured.io/api-reference/workflow/workflows#vlm-strategy)
+  --vlm-model     Specify the VLM model when using
+                  (see: https://docs.unstructured.io/api-reference/workflow/workflows#vlm-strategy)
+  --tables        Enable table extraction: tables are represented as html in metadata
+  --images        Include base64images in json
+  --coordinates   Include coordinates in the output
+  --trace         Enable trace logging for debugging, useful to cut and paste the executed curl call
+  --verbose       Enable verbose logging including printing first 8 elements to stdout
+  --s3            Write the resulting output to s3 (like a pastebin)
+  --write-html    Convert JSON output to HTML. Set the env var $UNST_WRITE_HTML to skip providing this option.
+  --open-html     Automatically open HTML output in browser (macOS only) if --write-html. 
+                  Set the env var UNST_AUTO_OPEN_HTML=true to skip providing this option.
+  --help          Display this help and exit.
+
+
+Arguments:
+  <file>          File to send to the API.
+
+If running against an API instance other than hosted Unstructured paid API (or --freemium),
+set the enviornment variable UNST_API_ENDPOINT.
+
+The script requires a <file>, the document to post to the Unstructured API.
+The .json result is written to ~/tmp/unst-outputs/ -- this path is echoed and copied to your clipboard.
+'
+
+if [ "$#" -eq 0 ]; then
+  echo "$USAGE_MESSAGE"
+  exit 1
+fi
+
+IMAGE_BLOCK_TYPES=${IMAGE_BLOCK_TYPES:-'"image", "table"'}
+API_KEY=${UNST_API_KEY:-""}
+TMP_DOWNLOADS_DIR=${UNST_SCRIPT_DOWNLOADS_DIR:-"$HOME/tmp/unst-downloads"}
+TMP_OUTPUTS_DIR=${UNST_SCRIPT_JSON_OUTPUTS_DIR:-"$HOME/tmp/unst-outputs"}
+# only applicable if writing .json output files to S3 when using --s3, e.g. s3://bucket-name/path/
+S3_URI_PREFIX=${UNST_S3_JSON_OUTPUT_URI:-""}
+# e.g. us-east-2, used to provide http links for above location
+S3_REGION=${UNST_S3_JSON_OUTPUT_REGION:-""}
+
+mkdir -p "$TMP_DOWNLOADS_DIR"
+mkdir -p "$TMP_OUTPUTS_DIR"
+
+copy_to_clipboard() {
+  if [ "$(uname)" == "Darwin" ]; then
+    # Join all arguments into a single string and copy to clipboard
+    echo "$*" | pbcopy
+    echo "copied to clipboard!"
+  fi
+  # TODO: add clipboard support for other OS's
+}
+
+HI_RES=false
+FAST=false
+OCR_ONLY=false
+VLM=false
+STRATEGY=""
+VERBOSE=false
+TRACE=false
+COORDINATES=false
+FREEMIUM=false
+TABLES=true
+IMAGES=false
+S3=""
+WRITE_HTML=${UNST_WRITE_HTML:-false}
+OPEN_HTML=${UNST_AUTO_OPEN_HTML:-false}
+VLM_PROVIDER=""
+VLM_MODEL=""
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+  --hi-res)
+    HI_RES=true
+    shift
+    ;;
+  --fast)
+    FAST=true
+    shift
+    ;;
+  --ocr-only)
+    OCR_ONLY=true
+    shift
+    ;;
+  --vlm)
+    VLM=true
+    shift
+    ;;
+  --vlm-provider)
+    if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+      VLM_PROVIDER=$2
+      shift 2
+    else
+      echo "Error: Argument for $1 is missing" >&2
+      exit 1
+    fi
+    ;;
+  --vlm-model)
+    if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+      VLM_MODEL=$2
+      shift 2
+    else
+      echo "Error: Argument for $1 is missing" >&2
+      exit 1
+    fi
+    ;;
+  --trace)
+    TRACE=true
+    shift
+    ;;
+  --verbose)
+    VERBOSE=true
+    shift
+    ;;
+  --s3)
+    S3=true
+    shift
+    ;;
+  --write-html)
+    WRITE_HTML=true
+    shift
+    ;;
+  --open-html)
+    OPEN_HTML=true
+    shift
+    ;;
+  --tables)
+    TABLES=true
+    shift
+    ;;
+  --images)
+    IMAGES=true
+    shift
+    ;;
+  --coordinates)
+    COORDINATES=true
+    shift
+    ;;
+  --freemium)
+    FREEMIUM=true
+    shift
+    ;;
+  --api-key)
+    if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+      API_KEY=$2
+      shift 2
+    else
+      echo "Error: Argument for $1 is missing" >&2
+      exit 1
+    fi
+    ;;
+  --help)
+    echo "$USAGE_MESSAGE"
+    exit 0
+    ;;
+  *)
+    INPUT="$1"
+    shift
+    ;;
+  esac
+done
+
+if [ -z "$INPUT" ]; then
+  echo "Error: File or URL argument is missing."
+  exit 1
+fi
+
+# Check for strategy conflicts after all arguments are processed
+STRATEGY_COUNT=0
+$HI_RES && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+$FAST && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+$OCR_ONLY && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+$VLM && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+
+if [ "$STRATEGY_COUNT" -gt 1 ]; then
+  echo "Error: Only one strategy option (--hi-res, --fast, --ocr-only, --vlm) can be specified at a time."
+  exit 1
+fi
+
+# Check if vlm-provider or vlm-model are provided without --vlm
+if { [ -n "$VLM_PROVIDER" ] || [ -n "$VLM_MODEL" ]; } && ! $VLM; then
+  echo "Error: --vlm-provider or --vlm-model can only be used with --vlm strategy."
+  exit 1
+fi
+
+if $TRACE; then
+  set -x
+fi
+
+if [[ "$INPUT" =~ ^https?:// ]]; then
+  FILENAME=$(basename "$INPUT")
+  if $VERBOSE; then echo "Downloading $FILENAME $INPUT to "; fi
+  INPUT_FILEPATH=${TMP_DOWNLOADS_DIR}/${FILENAME}
+  curl -q -o "${OUTPUT_FILEPATH}" "$INPUT"
+  echo "Downloaded file to ${OUTPUT_FILEPATH}"
+else
+  FILENAME=$(basename "$INPUT")
+  INPUT_FILEPATH=${INPUT}
+fi
+
+if $FREEMIUM; then
+  API_ENDPOINT="https://api.unstructured.io/general/v0/general"
+else
+  API_ENDPOINT=${UNST_API_ENDPOINT:-"https://api.unstructuredapp.io/general/v0/general"}
+fi
+
+if $HI_RES; then
+  if $VERBOSE; then echo "Sending API request with hi_res strategy"; fi
+  STRATEGY="-hi-res"
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=hi_res")
+elif $FAST; then
+  if $VERBOSE; then echo "Sending API request with fast strategy"; fi
+  STRATEGY="-fast"
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=fast")
+elif $OCR_ONLY; then
+  STRATEGY="-ocr-only"
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=ocr_only")
+elif $VLM; then
+  if $VERBOSE; then echo "Sending API request with vlm strategy"; fi
+  STRATEGY="-vlm"
+  # Add provider and model to filename if specified
+  if [ -n "$VLM_PROVIDER" ] && [ -n "$VLM_MODEL" ]; then
+    STRATEGY="-vlm-${VLM_PROVIDER}-${VLM_MODEL}"
+  elif [ -n "$VLM_PROVIDER" ]; then
+    STRATEGY="-vlm-${VLM_PROVIDER}"
+  elif [ -n "$VLM_MODEL" ]; then
+    STRATEGY="-vlm-model-${VLM_MODEL}"
+  fi
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=vlm")
+  if [ -n "$VLM_PROVIDER" ]; then
+    CURL_STRATEGY+=(-F "vlm_model_provider=$VLM_PROVIDER")
+  fi
+  if [ -n "$VLM_MODEL" ]; then
+    CURL_STRATEGY+=(-F "vlm_model=$VLM_MODEL")
+  fi
+else
+  if $VERBOSE; then echo "Sending API request WITHOUT a strategy"; fi
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=()
+fi
+
+CURL_API_KEY=()
+[[ -n "$API_KEY" ]] && CURL_API_KEY=(-H "unstructured-api-key: $API_KEY")
+CURL_COORDINATES=()
+[[ "$COORDINATES" == "true" ]] && CURL_COORDINATES=(-F "coordinates=true")
+CURL_TABLES=()
+[[ "$TABLES" == "true" ]] && CURL_TABLES=(-F "skip_infer_table_types='[]'")
+CURL_IMAGES=()
+[[ "$IMAGES" == "true" ]] && CURL_IMAGES=(-F "extract_image_block_types=[$IMAGE_BLOCK_TYPES]")
+
+curl -q -X 'POST' \
+  "$API_ENDPOINT" \
+  "${CURL_API_KEY[@]}" -H 'accept: application/json' \
+  -H 'Content-Type: multipart/form-data' \
+  "${CURL_STRATEGY[@]}" "${CURL_COORDINATES[@]}" "${CURL_TABLES[@]}" "${CURL_IMAGES[@]}" -F "files=@${INPUT_FILEPATH}" \
+  -o "${JSON_OUTPUT_FILEPATH}"
+
+JSON_FILE_SIZE=$(wc -c <"${JSON_OUTPUT_FILEPATH}")
+if [ "$JSON_FILE_SIZE" -lt 10 ]; then
+  echo "Error: JSON file ${JSON_OUTPUT_FILEPATH} has no elements."
+  cat "$JSON_OUTPUT_FILEPATH"
+  exit 1
+else
+  # shellcheck disable=SC2046
+  if $VERBOSE; then
+    echo "first 8 elements: "
+    jq '.[0:8]' "${JSON_OUTPUT_FILEPATH}"
+  fi
+  # shellcheck disable=SC2046
+  echo "total number of elements: " $(jq 'length' "${JSON_OUTPUT_FILEPATH}")
+fi
+echo "JSON Output file: ${JSON_OUTPUT_FILEPATH}"
+
+# Convert JSON to HTML if requested
+if [ "$WRITE_HTML" = true ]; then
+  HTML_OUTPUT_FILEPATH=${JSON_OUTPUT_FILEPATH%.json}.html
+
+  if $VLM; then
+    # VLM output has all metadata.text_as_html fields defined, so
+    # create HTML directly from the metadata.text_as_html fields
+    {
+      echo "<!DOCTYPE html>"
+      echo "<html>"
+      echo "<head>"
+      echo "  <meta charset=\"UTF-8\">"
+      echo "  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">"
+      echo "  <title>${FILENAME}</title>"
+      echo "  <style>"
+      echo "    body { font-family: Arial, sans-serif; line-height: 1.6; margin: 20px; }"
+      echo "  </style>"
+      echo "</head>"
+      echo "<body>"
+      jq -r 'map(.metadata.text_as_html) | join("\n")' "${JSON_OUTPUT_FILEPATH}"
+      echo "</body>"
+      echo "</html>"
+    } >"${HTML_OUTPUT_FILEPATH}"
+    echo "HTML written directly from metadata.text_as_html fields to: ${HTML_OUTPUT_FILEPATH}"
+  else
+    # most elements will not have metadata.text_as_html defined (by design on Table elements do),
+    # so use the unstructured library's python script for the conversion.
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    PYTHONPATH="${SCRIPT_DIR}/../.." python3 "${SCRIPT_DIR}/../convert/elements_json_to_format.py" "${JSON_OUTPUT_FILEPATH}" --outdir "${TMP_OUTPUTS_DIR}"
+    echo "HTML written using Python script to: ${HTML_OUTPUT_FILEPATH}"
+  fi
+
+  # Open HTML file in browser if requested and on macOS
+  if [ "$OPEN_HTML" = true ] && [ "$(uname)" == "Darwin" ]; then
+    open "${HTML_OUTPUT_FILEPATH}"
+  fi
+fi
+
+# write .json output to s3 location
+if [ -n "$S3" ]; then
+
+  if [ -z "$S3_URI_PREFIX" ]; then
+    echo
+    echo "You must define your s3 output location in the env var UNST_S3_JSON_OUTPUT_URI"
+    echo "e.g. UNST_S3_JSON_OUTPUT_URI='s3://bucket/path/'"
+    exit 0
+  elif [ -z "$S3_REGION" ]; then
+    echo
+    echo "You must define your s3 region in the env var UNST_S3_JSON_OUTPUT_REGION"
+    echo "e.g. UNST_S3_JSON_OUTPUT_REGION=us-west-2"
+    exit 0
+  fi
+
+  SHA_SUM_PREFIX=$(sha256sum "${JSON_OUTPUT_FILEPATH}" | cut -c1-7)
+  CURRENT_TIMESTAMP=$(date -u +%s)
+  APR27_2023_TIMESTAMP=$(date -u -d "2023-04-27 00:00:00" +%s)
+  TENS_OF_SECS_SINCE_APR27_2023=$(((CURRENT_TIMESTAMP - APR27_2023_TIMESTAMP) / 10))
+
+  S3_UPLOAD_PATH="${S3_URI_PREFIX}${TENS_OF_SECS_SINCE_APR27_2023}-${SHA_SUM_PREFIX}${STRATEGY}/${FILENAME}.json"
+  if $VERBOSE; then echo "Uploading JSON to S3"; fi
+  aws s3 cp "${JSON_OUTPUT_FILEPATH}" "$S3_UPLOAD_PATH"
+
+  BUCKET=$(echo "$S3_UPLOAD_PATH" | cut -d/ -f3)
+  KEY=$(echo "$S3_UPLOAD_PATH" | cut -d/ -f4-)
+  HTTPS_URL="https://${BUCKET}.s3.us-east-2.amazonaws.com/${KEY}"
+
+  echo "s3 location: ${S3_UPLOAD_PATH}"
+  echo "link: $HTTPS_URL"
+  copy_to_clipboard "$HTTPS_URL"
+else
+  copy_to_clipboard "${JSON_OUTPUT_FILEPATH}"
+fi

scripts/version-sync.sh

@@ -0,0 +1,171 @@
+#!/usr/bin/env bash
+
+set -u
+
+function usage {
+  echo "Usage: $(basename "$0") [-c] -f FILE_TO_CHANGE REPLACEMENT_FORMAT [-f FILE_TO_CHANGE REPLACEMENT_FORMAT ...]" 2>&1
+  echo 'Synchronize files to latest version in source file'
+  echo '   -s              Specifies source file for version (default is CHANGELOG.md)'
+  echo '   -f              Specifies a file to change and the format for searching and replacing versions'
+  echo '                       FILE_TO_CHANGE is the file to be updated/checked for updates'
+  echo '                       REPLACEMENT_FORMAT is one of (semver, release, api-release)'
+  echo '                           semver indicates to look for a full semver version and replace with the latest full version'
+  echo '                           release indicates to look for a release semver version (x.x.x) and replace with the latest release version'
+  echo '                           api-release indicates to look for a release semver version in the context of an api route and replace with the latest release version'
+  echo '   -c              Compare versions and output proposed changes without changing anything.'
+}
+
+function getopts-extra() {
+  declare -i i=1
+  # if the next argument is not an option, then append it to array OPTARG
+  while [[ ${OPTIND} -le $# && ${!OPTIND:0:1} != '-' ]]; do
+    OPTARG[i]=${!OPTIND}
+    ((i += 1))
+    ((OPTIND += 1))
+  done
+}
+
+# Parse input options
+declare CHECK=0
+declare SOURCE_FILE="CHANGELOG.md"
+declare -a FILES_TO_CHECK=()
+declare -a REPLACEMENT_FORMATS=()
+declare args
+declare OPTIND OPTARG opt
+while getopts ":hcs:f:" opt; do
+  case $opt in
+  h)
+    usage
+    exit 0
+    ;;
+  c)
+    CHECK=1
+    ;;
+  s)
+    SOURCE_FILE="$OPTARG"
+    ;;
+  f)
+    getopts-extra "$@"
+    args=("${OPTARG[@]}")
+    # validate length of args, should be 2
+    if [ ${#args[@]} -eq 2 ]; then
+      FILES_TO_CHECK+=("${args[0]}")
+      REPLACEMENT_FORMATS+=("${args[1]}")
+    else
+      echo "Exactly 2 arguments must follow -f option." >&2
+      exit 1
+    fi
+    ;;
+  \?)
+    echo "Invalid option: -$OPTARG." >&2
+    usage
+    exit 1
+    ;;
+  esac
+done
+
+# Parse REPLACEMENT_FORMATS
+RE_SEMVER_FULL="(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)(-((0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*)(\.(0|[1-9][0-9]*|[0-9]*[a-zA-Z-][0-9a-zA-Z-]*))*))?(\+([0-9a-zA-Z-]+(\.[0-9a-zA-Z-]+)*))?"
+RE_RELEASE="(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)"
+RE_API_RELEASE="v(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)\.(0|[1-9][0-9]*)"
+# Pull out semver appearing earliest in SOURCE_FILE.
+LAST_VERSION=$(grep -o -m 1 -E "${RE_SEMVER_FULL}" "$SOURCE_FILE")
+LAST_RELEASE=$(grep -o -m 1 -E "${RE_RELEASE}($|[^-+])" "$SOURCE_FILE" | grep -o -m 1 -E "${RE_RELEASE}")
+LAST_API_RELEASE="v$(grep -o -m 1 -E "${RE_RELEASE}($|[^-+])$" "$SOURCE_FILE" | grep -o -m 1 -E "${RE_RELEASE}")"
+declare -a RE_SEMVERS=()
+declare -a UPDATED_VERSIONS=()
+for i in "${!REPLACEMENT_FORMATS[@]}"; do
+  REPLACEMENT_FORMAT=${REPLACEMENT_FORMATS[$i]}
+  case $REPLACEMENT_FORMAT in
+  semver)
+    RE_SEMVERS+=("$RE_SEMVER_FULL")
+    UPDATED_VERSIONS+=("$LAST_VERSION")
+    ;;
+  release)
+    RE_SEMVERS+=("$RE_RELEASE")
+    UPDATED_VERSIONS+=("$LAST_RELEASE")
+    ;;
+  api-release)
+    RE_SEMVERS+=("$RE_API_RELEASE")
+    UPDATED_VERSIONS+=("$LAST_API_RELEASE")
+    ;;
+  *)
+    echo "Invalid replacement format: \"${REPLACEMENT_FORMAT}\". Use semver, release, or api-release" >&2
+    exit 1
+    ;;
+  esac
+done
+
+if [ -z "$LAST_VERSION" ]; then
+  # No match to semver regex in SOURCE_FILE, so no version to go from.
+  printf "Error: Unable to find latest version from %s.\n" "$SOURCE_FILE"
+  exit 1
+fi
+
+# Search files in FILES_TO_CHECK and change (or get diffs)
+declare FAILED_CHECK=0
+
+git fetch origin main
+MAIN_VERSION=$(git show origin/main:unstructured/__version__.py | grep -o -m 1 -E "${RE_SEMVER_FULL}")
+MAIN_IS_RELEASE=false
+[[ $MAIN_VERSION != *"-dev"* ]] && MAIN_IS_RELEASE=true
+CURRENT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+
+for i in "${!FILES_TO_CHECK[@]}"; do
+  FILE_TO_CHANGE=${FILES_TO_CHECK[$i]}
+  RE_SEMVER=${RE_SEMVERS[$i]}
+  UPDATED_VERSION=${UPDATED_VERSIONS[$i]}
+  FILE_VERSION=$(grep -o -m 1 -E "${RE_SEMVER}" "$FILE_TO_CHANGE")
+
+  if [ -z "$FILE_VERSION" ]; then
+    # No match to semver regex in VERSIONFILE, so nothing to replace
+    printf "Error: No semver version found in file %s.\n" "$FILE_TO_CHANGE"
+    exit 1
+  else
+    if [[ "$MAIN_IS_RELEASE" == true && "$UPDATED_VERSION" == "$MAIN_VERSION" && "$CURRENT_BRANCH" != "main" ]]; then
+      # Only one commit should be associated with a particular non-dev version
+      if [[ "$CHECK" == 1 ]]; then
+        printf "Error: there is already a commit associated with version %s.\n" "$MAIN_VERSION"
+        exit 1
+      else
+        printf "Warning: there is already a commit associated with version %s.\n" "$MAIN_VERSION"
+      fi
+    fi
+
+    # Replace semver in VERSIONFILE with semver obtained from SOURCE_FILE
+    TMPFILE=$(mktemp /tmp/new_version.XXXXXX)
+    # Check sed version, exit if version < 4.3
+    if ! sed --version >/dev/null 2>&1; then
+      CURRENT_VERSION=1.archaic
+    else
+      CURRENT_VERSION=$(sed --version | head -n1 | cut -d" " -f4)
+    fi
+    REQUIRED_VERSION="4.3"
+    if [ "$(printf '%s\n' "$REQUIRED_VERSION" "$CURRENT_VERSION" | sort -V | head -n1)" != "$REQUIRED_VERSION" ]; then
+      echo "sed version must be >= ${REQUIRED_VERSION}" && exit 1
+    fi
+    sed -E -r "s/$RE_SEMVER/$UPDATED_VERSION/" "$FILE_TO_CHANGE" >"$TMPFILE"
+    if [ $CHECK == 1 ]; then
+      DIFF=$(diff "$FILE_TO_CHANGE" "$TMPFILE")
+      if [ -z "$DIFF" ]; then
+        printf "version sync would make no changes to %s.\n" "$FILE_TO_CHANGE"
+        rm "$TMPFILE"
+      else
+        FAILED_CHECK=1
+        printf "version sync would make the following changes to %s:\n%s\n" "$FILE_TO_CHANGE" "$DIFF"
+        rm "$TMPFILE"
+      fi
+    else
+      cp "$TMPFILE" "$FILE_TO_CHANGE"
+      rm "$TMPFILE"
+    fi
+  fi
+done
+
+# Exit with code determined by whether changes were needed in a check.
+if [ ${FAILED_CHECK} -ne 0 ]; then
+  printf "\nVersions are out of sync! See above for diffs.\n"
+  exit 1
+else
+  exit 0
+fi

scripts/weaviate-test-helpers/create-weaviate-instance.sh

@@ -0,0 +1,13 @@
+#!/usr/bin/env bash
+
+set -e
+
+SCRIPT_DIR=$(dirname "$(realpath "$0")")
+
+# Create the Weaviate instance
+docker compose version
+docker compose -f "$SCRIPT_DIR"/docker-compose.yml up --wait
+docker compose -f "$SCRIPT_DIR"/docker-compose.yml ps
+
+echo "Instance is live."
+"$SCRIPT_DIR"/create_schema.py

scripts/weaviate-test-helpers/create_schema.py

@@ -0,0 +1,21 @@
+#!/usr/bin/env python3
+
+import json
+import os
+
+import weaviate
+
+weaviate_host_url = os.getenv("WEAVIATE_HOST_URL", "http://localhost:8080")
+class_name = os.getenv("WEAVIATE_CLASS_NAME", "Elements")
+new_class = None
+
+with open("./scripts/weaviate-test-helpers/elements.json") as f:
+    new_class = json.load(f)
+
+client = weaviate.Client(
+    url=weaviate_host_url,
+)
+
+if client.schema.exists(class_name):
+    client.schema.delete_class(class_name)
+client.schema.create_class(new_class)

scripts/weaviate-test-helpers/docker-compose.yml

@@ -0,0 +1,21 @@
+version: '3.4'
+services:
+  weaviate:
+    command:
+    - --host
+    - 0.0.0.0
+    - --port
+    - '8080'
+    - --scheme
+    - http
+    image: semitechnologies/weaviate:1.35.3
+    ports:
+    - 8080:8080
+    restart: on-failure:0
+    environment:
+      QUERY_DEFAULTS_LIMIT: 25
+      AUTHENTICATION_ANONYMOUS_ACCESS_ENABLED: 'true'
+      PERSISTENCE_DATA_PATH: '/var/lib/weaviate'
+      DEFAULT_VECTORIZER_MODULE: 'none'
+      ENABLE_MODULES: ''
+      CLUSTER_HOSTNAME: 'node1'

scripts/weaviate-test-helpers/elements.json

@@ -0,0 +1,414 @@
+{
+    "class": "Elements",
+    "invertedIndexConfig": {
+        "bm25": {
+            "b": 0.75,
+            "k1": 1.2
+        },
+        "cleanupIntervalSeconds": 60,
+        "stopwords": {
+            "additions": null,
+            "preset": "en",
+            "removals": null
+        }
+    },
+    "multiTenancyConfig": {
+        "enabled": false
+    },
+    "properties": [
+        {
+            "dataType": [
+                "text"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": true,
+            "name": "element_id",
+            "tokenization": "word"
+        },
+        {
+            "dataType": [
+                "text"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": true,
+            "name": "text",
+            "tokenization": "word"
+        },
+        {
+            "dataType": [
+                "text"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": true,
+            "name": "type",
+            "tokenization": "word"
+        },
+        {
+            "dataType": [
+                "object"
+            ],
+            "indexFilterable": true,
+            "indexSearchable": false,
+            "name": "metadata",
+            "nestedProperties": [
+                {
+                    "dataType": [
+                        "int"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": false,
+                    "name": "category_depth"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "parent_id",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "attached_to_filename",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "filetype",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "date"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": false,
+                    "name": "last_modified"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "file_directory",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "filename",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "object"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": false,
+                    "name": "data_source",
+                    "nestedProperties": [
+                        {
+                            "dataType": [
+                                "text"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": true,
+                            "name": "url",
+                            "tokenization": "word"
+                        },
+                        {
+                            "dataType": [
+                                "text"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": true,
+                            "name": "version",
+                            "tokenization": "word"
+                        },
+                        {
+                            "dataType": [
+                                "date"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": false,
+                            "name": "date_created"
+                        },
+                        {
+                            "dataType": [
+                                "date"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": false,
+                            "name": "date_modified"
+                        },
+                        {
+                            "dataType": [
+                                "date"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": false,
+                            "name": "date_processed"
+                        },
+                        {
+                            "dataType": [
+                                "text"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": true,
+                            "name": "record_locator",
+                            "tokenization": "word"
+                        },
+                        {
+                            "dataType": [
+                                "text"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": true,
+                            "name": "permissions_data",
+                            "tokenization": "word"
+                        }
+
+                    ]
+                },
+                {
+                    "dataType": [
+                        "object"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": false,
+                    "name": "coordinates",
+                    "nestedProperties": [
+                        {
+                            "dataType": [
+                                "text"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": true,
+                            "name": "system",
+                            "tokenization": "word"
+                        },
+                        {
+                            "dataType": [
+                                "number"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": false,
+                            "name": "layout_width"
+                        },
+                        {
+                            "dataType": [
+                                "number"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": false,
+                            "name": "layout_height"
+                        },
+                        {
+                            "dataType": [
+                                "text"
+                            ],
+                            "indexFilterable": true,
+                            "indexSearchable": true,
+                            "name": "points",
+                            "tokenization": "word"
+                        }
+                    ]
+                },
+                {
+                    "dataType": [
+                        "text[]"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "languages",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": false,
+                    "name": "page_number"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "page_name",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "url",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "links",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text[]"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "link_urls",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text[]"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "link_texts",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "sent_from",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "sent_to",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "subject",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "section",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "header_footer_type",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text[]"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "emphasized_text_contents",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text[]"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "emphasized_text_tags",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "text"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": true,
+                    "name": "text_as_html",
+                    "tokenization": "word"
+                },
+                {
+                    "dataType": [
+                        "number"
+                    ],
+                    "indexFilterable": true,
+                    "indexSearchable": false,
+                    "name": "detection_class_prob"
+                }
+            ]
+        }
+    ],
+    "replicationConfig": {
+        "factor": 1
+    },
+    "shardingConfig": {
+        "virtualPerPhysical": 128,
+        "desiredCount": 1,
+        "actualCount": 1,
+        "desiredVirtualCount": 128,
+        "actualVirtualCount": 128,
+        "key": "_id",
+        "strategy": "hash",
+        "function": "murmur3"
+    },
+    "vectorIndexConfig": {
+        "skip": false,
+        "cleanupIntervalSeconds": 300,
+        "maxConnections": 64,
+        "efConstruction": 128,
+        "ef": -1,
+        "dynamicEfMin": 100,
+        "dynamicEfMax": 500,
+        "dynamicEfFactor": 8,
+        "vectorCacheMaxObjects": 1000000000000,
+        "flatSearchCutoff": 40000,
+        "distance": "cosine",
+        "pq": {
+            "enabled": false,
+            "bitCompression": false,
+            "segments": 0,
+            "centroids": 256,
+            "trainingLimit": 100000,
+            "encoder": {
+                "type": "kmeans",
+                "distribution": "log-normal"
+            }
+        }
+    },
+    "vectorIndexType": "hnsw",
+    "vectorizer": "none"
+}