chore: import upstream snapshot with attribution

scripts/user/process-pdf-parallel-through-api.sh

@@ -0,0 +1,125 @@
+#!/usr/bin/env bash
+
+# Usage: ./process-pdf-parallel-through-api.sh filename.pdf
+
+set -eu -o pipefail
+
+if [ $# -ne 1 ]; then
+  echo "Processes a single PDF through the Unstructured API by breaking it into smaller splits that are processed concurrently."
+  echo
+  echo "Usage: $0 <pdf_filename>"
+  echo "Please provide a PDF filename as the first argument."
+  echo
+  echo "Optionally, set the following env vars: "
+  echo
+  echo "* STRATEGY (default hi_res)"
+  echo "* BATCH_SIZE (default 30) as the number of parts (AKA splits) to process in parallel"
+  echo "* PDF_SPLIT_PAGE_SIZE (default 10) as the number of pages per split"
+  echo
+  echo "BATCH_SIZE=20 PDF_SPLIT_PAGE_SIZE=6 STRATEGY=hi_res ./process-pdf-parallel-through-api.sh example-docs/pdf/layout-parser-paper.pdf"
+  exit 1
+fi
+
+ALLOWED_STRATEGIES=("hi_res" "fast" "auto")
+
+# Validate STRATEGY environment variable if it's set
+if [ -n "${STRATEGY:-}" ] && [[ ! " ${ALLOWED_STRATEGIES[*]} " =~ ${STRATEGY} ]]; then
+  echo "Error: STRATEGY must be one of ${ALLOWED_STRATEGIES[*]}" >&2
+  exit 1
+fi
+
+# Check if UNST_API_KEY is set
+if [ -z "${UNST_API_KEY}" ]; then
+  echo "Error: UNST_API_KEY is not set or is empty" >&2
+  exit 1
+fi
+
+PDF_FILE="$1"
+DEFAULT_SPLIT_SIZE=10
+SPLIT_SIZE=${PDF_SPLIT_PAGE_SIZE:-$DEFAULT_SPLIT_SIZE}
+PDF_NAME=$(basename "$PDF_FILE" .pdf)
+DEFAULT_DIR="$HOME/tmp/pdf-splits"
+PDF_SPLITS_DIR="${PDF_SPLITS_DIR:-$DEFAULT_DIR}"
+MD5_SUM=$(md5sum "$PDF_FILE" | awk '{ print $1 }')
+PDF_DIR="$PDF_SPLITS_DIR/$PDF_NAME-${MD5_SUM}_split-${SPLIT_SIZE}"
+PDF_OUTPUT_DIR="$PDF_SPLITS_DIR/${PDF_NAME}-output-${MD5_SUM}_split-${SPLIT_SIZE}_strat-${STRATEGY}"
+
+SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+
+# Check if PDF parts directory exists
+if [ ! -d "$PDF_DIR" ]; then
+  "$SCRIPT_DIR/split-pdf.sh" "$PDF_FILE"
+fi
+
+# Create output directory if it does not exist
+mkdir -p "$PDF_OUTPUT_DIR"
+
+incomplete=0 # Flag to track incomplete processing
+
+# Function to process a single PDF part file
+process_file_part() {
+  local file="$1"
+  local STARTING_PAGE_NUMBER="$2"
+  local OUTPUT_JSON="$3"
+
+  if [ -f "$OUTPUT_JSON" ]; then
+    echo "Skipping processing for $OUTPUT_JSON as it already exists."
+    return
+  fi
+
+  curl -q -X POST https://api.unstructuredapp.io/general/v0/general \
+    -H "unstructured-api-key: $UNST_API_KEY" \
+    -H 'accept: application/json' \
+    -H 'Content-Type: multipart/form-data' \
+    -F strategy="${STRATEGY:-hi_res}" \
+    -F 'skip_infer_table_types="[]"' \
+    -F starting_page_number="$STARTING_PAGE_NUMBER" \
+    -F files=@"$file;filename=$PDF_FILE" \
+    -o "$OUTPUT_JSON"
+
+  # Verify JSON content
+  if ! jq -e 'if type=="array" then all(.[]; type=="object" or length==0) else empty end' "$OUTPUT_JSON" >/dev/null; then
+    echo "Invalid JSON structure in $OUTPUT_JSON (contents below), deleting file."
+    cat "$OUTPUT_JSON"
+    rm "$OUTPUT_JSON"
+    incomplete=1
+  else
+    echo "Valid JSON output created: $OUTPUT_JSON"
+  fi
+}
+
+# Function to process a batch of files
+process_batch() {
+  for file in "$@"; do
+    local START_PAGE
+    START_PAGE=$(echo "$file" | sed -n 's/.*_pages_\([0-9]*\)_to_[0-9]*.pdf/\1/p')
+    local END_PAGE=
+    END_PAGE=$(echo "$file" | sed -n 's/.*_pages_[0-9]*_to_\([0-9]*\).pdf/\1/p')
+    local OUTPUT_JSON="$PDF_OUTPUT_DIR/${PDF_NAME}_pages_${START_PAGE}_to_${END_PAGE}.json"
+    process_file_part "$file" "$START_PAGE" "$OUTPUT_JSON" &
+  done
+  wait
+}
+
+# Read PDF parts into an array
+mapfile -t pdf_parts < <(find "$PDF_DIR" -name '*.pdf' -print)
+
+# Process PDF parts in batches of 30, by default
+batch_size=${BATCH_SIZE:-30}
+for ((i = 0; i < ${#pdf_parts[@]}; i += batch_size)); do
+  process_batch "${pdf_parts[@]:i:batch_size}"
+done
+
+# Determine the output filename based on whether processing was incomplete
+if [ "$incomplete" -eq 1 ]; then
+  combined_output_filename="${PDF_NAME}_incomplete_combined.json"
+  echo "WARNING! not all json parts were successfully processed. you may rerun this script"
+  echo "to attempt reprocessing those (failed to process) parts."
+else
+  combined_output_filename="${PDF_NAME}_combined.json"
+fi
+
+# Combine JSON outputs in numerical order
+find "$PDF_OUTPUT_DIR" -name '*.json' -print0 | sort -zV | xargs -0 jq -s 'add' >"$PDF_OUTPUT_DIR/$combined_output_filename"
+
+echo "Processing complete. Combined JSON saved to $PDF_OUTPUT_DIR/$combined_output_filename"

scripts/user/split-pdf.sh

@@ -0,0 +1,42 @@
+#!/usr/bin/env bash
+
+# Usage: ./split_pdf.sh filename.pdf
+
+set -e
+
+PDF_FILE="$1"
+DEFAULT_SPLIT_SIZE=5
+SPLIT_SIZE=${PDF_SPLIT_PAGE_SIZE:-$DEFAULT_SPLIT_SIZE}
+
+# Validate that SPLIT_SIZE is an integer
+if ! [[ "$SPLIT_SIZE" =~ ^[0-9]+$ ]]; then
+  echo "Error: PDF_SPLIT_PAGE_SIZE must be an integer."
+  exit 1
+fi
+
+DEFAULT_DIR="$HOME/tmp/pdf-splits"
+PDF_SPLITS_DIR="${PDF_SPLITS_DIR:-$DEFAULT_DIR}"
+PDF_NAME=$(basename "$PDF_FILE" .pdf)
+MD5_SUM=$(md5sum "$PDF_FILE" | awk '{ print $1 }')
+PDF_DIR="$PDF_SPLITS_DIR/$PDF_NAME-${MD5_SUM}_split-${SPLIT_SIZE}"
+
+# Create directory if it does not exist
+mkdir -p "$PDF_DIR"
+
+# Total number of pages
+TOTAL_PAGES=$(qpdf --show-npages "$PDF_FILE")
+
+# Split PDF into $SPLIT_SIZE-page chunks
+START_PAGE=1
+while [ "$START_PAGE" -le "$TOTAL_PAGES" ]; do
+  END_PAGE=$((START_PAGE + SPLIT_SIZE - 1))
+  if [ "$END_PAGE" -gt "$TOTAL_PAGES" ]; then
+    END_PAGE=$TOTAL_PAGES
+  fi
+  OUTPUT_FILE="$PDF_DIR/${PDF_NAME}_pages_${START_PAGE}_to_${END_PAGE}.pdf"
+  qpdf "$PDF_FILE" --pages . "$START_PAGE"-"$END_PAGE" -- "$OUTPUT_FILE"
+  echo "Created $OUTPUT_FILE"
+  START_PAGE=$((END_PAGE + 1))
+done
+
+echo "All parts have been saved to $PDF_DIR"

scripts/user/u-tables-inspect.sh

@@ -0,0 +1,70 @@
+#!/usr/bin/env bash
+
+# TODO:
+# * include the page number in output html filename, maybe all tables per page
+#   in one html page.
+
+set -e
+
+USAGE_MESSAGE="Usage: $0 <file>
+
+Requires an unstructured output .json file as the only argument.
+
+Each table in the file is saved as an individual html file and
+opened in Safari (if running on mac), providing a quick and easy
+way to see the structure and content of each Table element.
+"
+
+# Check for the presence of an argument
+if [ "$#" -ne 1 ]; then
+  echo "$USAGE_MESSAGE"
+  exit 1
+fi
+
+# The JSON file to be processed is the first argument
+JSON_FILE="$1"
+
+# Extract the basename without the extension
+BASE_NAME=$(basename "$JSON_FILE" .json)
+
+# Directory where the files will be saved
+TMP_TABLES_OUTPUTS_DIR="$HOME/tmp/tables-out"
+
+# Check if the directory exists, if not create it
+if [ ! -d "$TMP_TABLES_OUTPUTS_DIR" ]; then
+  mkdir -p "$TMP_TABLES_OUTPUTS_DIR"
+fi
+
+# Counter for the table files
+COUNTER=1
+
+# Parsing the JSON and creating HTML files
+jq -c '.[] | select(.type == "Table") | .metadata.text_as_html' "$JSON_FILE" | while read -r HTML_CONTENT; do
+  # Remove leading and trailing quotes from the JSON string
+  HTML_CONTENT=${HTML_CONTENT#\"}
+  HTML_CONTENT=${HTML_CONTENT%\"}
+  # add a border and padding to clearly see cell definition
+  # shellcheck disable=SC2001
+  HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/<table /<table border="1" cellpadding="10" /')
+  # shellcheck disable=SC2001
+  HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/<table>/<table border="1" cellpadding="10">/')
+  # add newlines for readability in the html
+  # shellcheck disable=SC2001
+  HTML_CONTENT=$(echo "$HTML_CONTENT" | sed 's/>\s*</>\n</g')
+
+  # Create filename based on the basename and counter
+  HTML_FILENAME="$TMP_TABLES_OUTPUTS_DIR/${BASE_NAME}-${COUNTER}.html"
+
+  # Increment the counter
+  COUNTER=$((COUNTER + 1))
+
+  # Save the HTML content to a file
+  echo "$HTML_CONTENT" >"$HTML_FILENAME"
+
+  if [ "$(uname)" == "Darwin" ]; then
+    # Open the file in a new browser window
+    open -a "Safari" "$HTML_FILENAME" &
+  else
+    echo "$HTML_FILENAME"
+  fi
+done

scripts/user/unstructured-get-json.sh

@@ -0,0 +1,367 @@
+#!/usr/bin/env bash
+
+# TODO's
+# * ability to set file type so that is not inferred by the unstructured api service
+#     e.g. "-F 'files=@foo.pdf;type=application/pdf'
+#
+
+set -e
+
+# shellcheck disable=SC2016
+USAGE_MESSAGE="Usage: $0 [options] <file>"'
+
+Options:
+  --api-key KEY   Specify the API key for authentication. Set the env var $UNST_API_KEY to skip providing this option.
+  --freemium      Use the free API rather paid API
+  --hi-res        hi_res strategy: Enable high-resolution processing, with layout segmentation and OCR
+  --fast          fast strategy: No OCR, just extract embedded text
+  --ocr-only      ocr_only strategy: Perform OCR (Optical Character Recognition) only. No layout segmentation.
+  --vlm           vlm strategy: Use Vision Language Model for processing
+  --vlm-provider  Specify the VLM model provider
+                  (see: https://docs.unstructured.io/api-reference/workflow/workflows#vlm-strategy)
+  --vlm-model     Specify the VLM model when using
+                  (see: https://docs.unstructured.io/api-reference/workflow/workflows#vlm-strategy)
+  --tables        Enable table extraction: tables are represented as html in metadata
+  --images        Include base64images in json
+  --coordinates   Include coordinates in the output
+  --trace         Enable trace logging for debugging, useful to cut and paste the executed curl call
+  --verbose       Enable verbose logging including printing first 8 elements to stdout
+  --s3            Write the resulting output to s3 (like a pastebin)
+  --write-html    Convert JSON output to HTML. Set the env var $UNST_WRITE_HTML to skip providing this option.
+  --open-html     Automatically open HTML output in browser (macOS only) if --write-html. 
+                  Set the env var UNST_AUTO_OPEN_HTML=true to skip providing this option.
+  --help          Display this help and exit.
+
+
+Arguments:
+  <file>          File to send to the API.
+
+If running against an API instance other than hosted Unstructured paid API (or --freemium),
+set the enviornment variable UNST_API_ENDPOINT.
+
+The script requires a <file>, the document to post to the Unstructured API.
+The .json result is written to ~/tmp/unst-outputs/ -- this path is echoed and copied to your clipboard.
+'
+
+if [ "$#" -eq 0 ]; then
+  echo "$USAGE_MESSAGE"
+  exit 1
+fi
+
+IMAGE_BLOCK_TYPES=${IMAGE_BLOCK_TYPES:-'"image", "table"'}
+API_KEY=${UNST_API_KEY:-""}
+TMP_DOWNLOADS_DIR=${UNST_SCRIPT_DOWNLOADS_DIR:-"$HOME/tmp/unst-downloads"}
+TMP_OUTPUTS_DIR=${UNST_SCRIPT_JSON_OUTPUTS_DIR:-"$HOME/tmp/unst-outputs"}
+# only applicable if writing .json output files to S3 when using --s3, e.g. s3://bucket-name/path/
+S3_URI_PREFIX=${UNST_S3_JSON_OUTPUT_URI:-""}
+# e.g. us-east-2, used to provide http links for above location
+S3_REGION=${UNST_S3_JSON_OUTPUT_REGION:-""}
+
+mkdir -p "$TMP_DOWNLOADS_DIR"
+mkdir -p "$TMP_OUTPUTS_DIR"
+
+copy_to_clipboard() {
+  if [ "$(uname)" == "Darwin" ]; then
+    # Join all arguments into a single string and copy to clipboard
+    echo "$*" | pbcopy
+    echo "copied to clipboard!"
+  fi
+  # TODO: add clipboard support for other OS's
+}
+
+HI_RES=false
+FAST=false
+OCR_ONLY=false
+VLM=false
+STRATEGY=""
+VERBOSE=false
+TRACE=false
+COORDINATES=false
+FREEMIUM=false
+TABLES=true
+IMAGES=false
+S3=""
+WRITE_HTML=${UNST_WRITE_HTML:-false}
+OPEN_HTML=${UNST_AUTO_OPEN_HTML:-false}
+VLM_PROVIDER=""
+VLM_MODEL=""
+
+while [[ "$#" -gt 0 ]]; do
+  case "$1" in
+  --hi-res)
+    HI_RES=true
+    shift
+    ;;
+  --fast)
+    FAST=true
+    shift
+    ;;
+  --ocr-only)
+    OCR_ONLY=true
+    shift
+    ;;
+  --vlm)
+    VLM=true
+    shift
+    ;;
+  --vlm-provider)
+    if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+      VLM_PROVIDER=$2
+      shift 2
+    else
+      echo "Error: Argument for $1 is missing" >&2
+      exit 1
+    fi
+    ;;
+  --vlm-model)
+    if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+      VLM_MODEL=$2
+      shift 2
+    else
+      echo "Error: Argument for $1 is missing" >&2
+      exit 1
+    fi
+    ;;
+  --trace)
+    TRACE=true
+    shift
+    ;;
+  --verbose)
+    VERBOSE=true
+    shift
+    ;;
+  --s3)
+    S3=true
+    shift
+    ;;
+  --write-html)
+    WRITE_HTML=true
+    shift
+    ;;
+  --open-html)
+    OPEN_HTML=true
+    shift
+    ;;
+  --tables)
+    TABLES=true
+    shift
+    ;;
+  --images)
+    IMAGES=true
+    shift
+    ;;
+  --coordinates)
+    COORDINATES=true
+    shift
+    ;;
+  --freemium)
+    FREEMIUM=true
+    shift
+    ;;
+  --api-key)
+    if [ -n "$2" ] && [ "${2:0:1}" != "-" ]; then
+      API_KEY=$2
+      shift 2
+    else
+      echo "Error: Argument for $1 is missing" >&2
+      exit 1
+    fi
+    ;;
+  --help)
+    echo "$USAGE_MESSAGE"
+    exit 0
+    ;;
+  *)
+    INPUT="$1"
+    shift
+    ;;
+  esac
+done
+
+if [ -z "$INPUT" ]; then
+  echo "Error: File or URL argument is missing."
+  exit 1
+fi
+
+# Check for strategy conflicts after all arguments are processed
+STRATEGY_COUNT=0
+$HI_RES && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+$FAST && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+$OCR_ONLY && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+$VLM && STRATEGY_COUNT=$((STRATEGY_COUNT + 1))
+
+if [ "$STRATEGY_COUNT" -gt 1 ]; then
+  echo "Error: Only one strategy option (--hi-res, --fast, --ocr-only, --vlm) can be specified at a time."
+  exit 1
+fi
+
+# Check if vlm-provider or vlm-model are provided without --vlm
+if { [ -n "$VLM_PROVIDER" ] || [ -n "$VLM_MODEL" ]; } && ! $VLM; then
+  echo "Error: --vlm-provider or --vlm-model can only be used with --vlm strategy."
+  exit 1
+fi
+
+if $TRACE; then
+  set -x
+fi
+
+if [[ "$INPUT" =~ ^https?:// ]]; then
+  FILENAME=$(basename "$INPUT")
+  if $VERBOSE; then echo "Downloading $FILENAME $INPUT to "; fi
+  INPUT_FILEPATH=${TMP_DOWNLOADS_DIR}/${FILENAME}
+  curl -q -o "${OUTPUT_FILEPATH}" "$INPUT"
+  echo "Downloaded file to ${OUTPUT_FILEPATH}"
+else
+  FILENAME=$(basename "$INPUT")
+  INPUT_FILEPATH=${INPUT}
+fi
+
+if $FREEMIUM; then
+  API_ENDPOINT="https://api.unstructured.io/general/v0/general"
+else
+  API_ENDPOINT=${UNST_API_ENDPOINT:-"https://api.unstructuredapp.io/general/v0/general"}
+fi
+
+if $HI_RES; then
+  if $VERBOSE; then echo "Sending API request with hi_res strategy"; fi
+  STRATEGY="-hi-res"
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=hi_res")
+elif $FAST; then
+  if $VERBOSE; then echo "Sending API request with fast strategy"; fi
+  STRATEGY="-fast"
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=fast")
+elif $OCR_ONLY; then
+  STRATEGY="-ocr-only"
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=ocr_only")
+elif $VLM; then
+  if $VERBOSE; then echo "Sending API request with vlm strategy"; fi
+  STRATEGY="-vlm"
+  # Add provider and model to filename if specified
+  if [ -n "$VLM_PROVIDER" ] && [ -n "$VLM_MODEL" ]; then
+    STRATEGY="-vlm-${VLM_PROVIDER}-${VLM_MODEL}"
+  elif [ -n "$VLM_PROVIDER" ]; then
+    STRATEGY="-vlm-${VLM_PROVIDER}"
+  elif [ -n "$VLM_MODEL" ]; then
+    STRATEGY="-vlm-model-${VLM_MODEL}"
+  fi
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=(-F "strategy=vlm")
+  if [ -n "$VLM_PROVIDER" ]; then
+    CURL_STRATEGY+=(-F "vlm_model_provider=$VLM_PROVIDER")
+  fi
+  if [ -n "$VLM_MODEL" ]; then
+    CURL_STRATEGY+=(-F "vlm_model=$VLM_MODEL")
+  fi
+else
+  if $VERBOSE; then echo "Sending API request WITHOUT a strategy"; fi
+  JSON_OUTPUT_FILEPATH=${TMP_OUTPUTS_DIR}/${FILENAME}${STRATEGY}.json
+  CURL_STRATEGY=()
+fi
+
+CURL_API_KEY=()
+[[ -n "$API_KEY" ]] && CURL_API_KEY=(-H "unstructured-api-key: $API_KEY")
+CURL_COORDINATES=()
+[[ "$COORDINATES" == "true" ]] && CURL_COORDINATES=(-F "coordinates=true")
+CURL_TABLES=()
+[[ "$TABLES" == "true" ]] && CURL_TABLES=(-F "skip_infer_table_types='[]'")
+CURL_IMAGES=()
+[[ "$IMAGES" == "true" ]] && CURL_IMAGES=(-F "extract_image_block_types=[$IMAGE_BLOCK_TYPES]")
+
+curl -q -X 'POST' \
+  "$API_ENDPOINT" \
+  "${CURL_API_KEY[@]}" -H 'accept: application/json' \
+  -H 'Content-Type: multipart/form-data' \
+  "${CURL_STRATEGY[@]}" "${CURL_COORDINATES[@]}" "${CURL_TABLES[@]}" "${CURL_IMAGES[@]}" -F "files=@${INPUT_FILEPATH}" \
+  -o "${JSON_OUTPUT_FILEPATH}"
+
+JSON_FILE_SIZE=$(wc -c <"${JSON_OUTPUT_FILEPATH}")
+if [ "$JSON_FILE_SIZE" -lt 10 ]; then
+  echo "Error: JSON file ${JSON_OUTPUT_FILEPATH} has no elements."
+  cat "$JSON_OUTPUT_FILEPATH"
+  exit 1
+else
+  # shellcheck disable=SC2046
+  if $VERBOSE; then
+    echo "first 8 elements: "
+    jq '.[0:8]' "${JSON_OUTPUT_FILEPATH}"
+  fi
+  # shellcheck disable=SC2046
+  echo "total number of elements: " $(jq 'length' "${JSON_OUTPUT_FILEPATH}")
+fi
+echo "JSON Output file: ${JSON_OUTPUT_FILEPATH}"
+
+# Convert JSON to HTML if requested
+if [ "$WRITE_HTML" = true ]; then
+  HTML_OUTPUT_FILEPATH=${JSON_OUTPUT_FILEPATH%.json}.html
+
+  if $VLM; then
+    # VLM output has all metadata.text_as_html fields defined, so
+    # create HTML directly from the metadata.text_as_html fields
+    {
+      echo "<!DOCTYPE html>"
+      echo "<html>"
+      echo "<head>"
+      echo "  <meta charset=\"UTF-8\">"
+      echo "  <meta name=\"viewport\" content=\"width=device-width, initial-scale=1.0\">"
+      echo "  <title>${FILENAME}</title>"
+      echo "  <style>"
+      echo "    body { font-family: Arial, sans-serif; line-height: 1.6; margin: 20px; }"
+      echo "  </style>"
+      echo "</head>"
+      echo "<body>"
+      jq -r 'map(.metadata.text_as_html) | join("\n")' "${JSON_OUTPUT_FILEPATH}"
+      echo "</body>"
+      echo "</html>"
+    } >"${HTML_OUTPUT_FILEPATH}"
+    echo "HTML written directly from metadata.text_as_html fields to: ${HTML_OUTPUT_FILEPATH}"
+  else
+    # most elements will not have metadata.text_as_html defined (by design on Table elements do),
+    # so use the unstructured library's python script for the conversion.
+    SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
+    PYTHONPATH="${SCRIPT_DIR}/../.." python3 "${SCRIPT_DIR}/../convert/elements_json_to_format.py" "${JSON_OUTPUT_FILEPATH}" --outdir "${TMP_OUTPUTS_DIR}"
+    echo "HTML written using Python script to: ${HTML_OUTPUT_FILEPATH}"
+  fi
+
+  # Open HTML file in browser if requested and on macOS
+  if [ "$OPEN_HTML" = true ] && [ "$(uname)" == "Darwin" ]; then
+    open "${HTML_OUTPUT_FILEPATH}"
+  fi
+fi
+
+# write .json output to s3 location
+if [ -n "$S3" ]; then
+
+  if [ -z "$S3_URI_PREFIX" ]; then
+    echo
+    echo "You must define your s3 output location in the env var UNST_S3_JSON_OUTPUT_URI"
+    echo "e.g. UNST_S3_JSON_OUTPUT_URI='s3://bucket/path/'"
+    exit 0
+  elif [ -z "$S3_REGION" ]; then
+    echo
+    echo "You must define your s3 region in the env var UNST_S3_JSON_OUTPUT_REGION"
+    echo "e.g. UNST_S3_JSON_OUTPUT_REGION=us-west-2"
+    exit 0
+  fi
+
+  SHA_SUM_PREFIX=$(sha256sum "${JSON_OUTPUT_FILEPATH}" | cut -c1-7)
+  CURRENT_TIMESTAMP=$(date -u +%s)
+  APR27_2023_TIMESTAMP=$(date -u -d "2023-04-27 00:00:00" +%s)
+  TENS_OF_SECS_SINCE_APR27_2023=$(((CURRENT_TIMESTAMP - APR27_2023_TIMESTAMP) / 10))
+
+  S3_UPLOAD_PATH="${S3_URI_PREFIX}${TENS_OF_SECS_SINCE_APR27_2023}-${SHA_SUM_PREFIX}${STRATEGY}/${FILENAME}.json"
+  if $VERBOSE; then echo "Uploading JSON to S3"; fi
+  aws s3 cp "${JSON_OUTPUT_FILEPATH}" "$S3_UPLOAD_PATH"
+
+  BUCKET=$(echo "$S3_UPLOAD_PATH" | cut -d/ -f3)
+  KEY=$(echo "$S3_UPLOAD_PATH" | cut -d/ -f4-)
+  HTTPS_URL="https://${BUCKET}.s3.us-east-2.amazonaws.com/${KEY}"
+
+  echo "s3 location: ${S3_UPLOAD_PATH}"
+  echo "link: $HTTPS_URL"
+  copy_to_clipboard "$HTTPS_URL"
+else
+  copy_to_clipboard "${JSON_OUTPUT_FILEPATH}"
+fi