chore: import upstream snapshot with attribution

scripts/airtable-test-helpers/component_ids.sh

@@ -0,0 +1,32 @@
+#!/usr/bin/env bash
+
+# Components below are all created using Airtable UI, however, in case they need
+# to be recreated, it is also possible to create them using the Web API.
+# pyairtable does not yet support creating these components (bases, tables).
+
+# For documentation on the Web API for creating bases, check:
+# https://airtable.com/developers/web/api/create-base
+
+# For creating lots of tables inside a base, check:
+# create_scale_test_components.sh
+
+LARGE_TABLE_BASE_ID="appQqieVsbxpwwD3i"
+LARGE_TABLE_TABLE_ID="tbll85GCfxED1OrvC"
+LARGE_BASE_BASE_ID="appjPRwoyawsapoGW"
+LARGE_WORKSPACE_BASE_ID_1="appSSCNWuIMjzeraO"
+LARGE_WORKSPACE_BASE_ID_2="appyvCsaHWn38RzFc"
+LARGE_WORKSPACE_BASE_ID_3="appbd8fkBv3AXj0Ab"
+LARGE_WORKSPACE_BASE_ID_4="appHEvCPnpfiAwjPE"
+LARGE_WORKSPACE_BASE_ID_5="appL9ND7LVWaItAmC"
+LARGE_WORKSPACE_BASE_ID_6="appOGnidMsh93yCQI"
+LARGE_WORKSPACE_BASE_ID_7="apps71HjvZRRgqHkz"
+LARGE_WORKSPACE_BASE_ID_8="appvDbw5f7jCQqdsr"
+LARGE_WORKSPACE_BASE_ID_9="appGFdtbLmqf2k8Ly"
+LARGE_WORKSPACE_BASE_ID_10="appTn61bfU8vCIkGf"
+LARGE_WORKSPACE_BASE_ID_11="app1c4CtIQ4ZToHIR"
+LARGE_WORKSPACE_BASE_ID_12="apphvDFg6OC7l1xwo"
+# shellcheck disable=SC2034
+LARGE_TEST_LIST_OF_PATHS="$LARGE_BASE_BASE_ID $LARGE_TABLE_BASE_ID $LARGE_WORKSPACE_BASE_ID_1 $LARGE_WORKSPACE_BASE_ID_2 $LARGE_WORKSPACE_BASE_ID_3 $LARGE_WORKSPACE_BASE_ID_4 $LARGE_WORKSPACE_BASE_ID_5 $LARGE_WORKSPACE_BASE_ID_6 $LARGE_WORKSPACE_BASE_ID_7 $LARGE_WORKSPACE_BASE_ID_8 $LARGE_WORKSPACE_BASE_ID_9 $LARGE_WORKSPACE_BASE_ID_10 $LARGE_WORKSPACE_BASE_ID_11 $LARGE_WORKSPACE_BASE_ID_12"
+
+export LARGE_TABLE_BASE_ID
+export LARGE_TABLE_TABLE_ID

scripts/airtable-test-helpers/create_scale_test_components.py

@@ -0,0 +1,82 @@
+import os
+
+# import pyairtable as pyair
+from pyairtable import Api
+
+from unstructured.logger import logger
+
+SCALE_TEST_NUMBER_OF_RECORDS = 20_000
+
+# Access token that has read and write permissions for the respective workspace
+token = os.environ["AIRTABLE_ACCESS_TOKEN_WRITE"]
+
+# You can find the IDs below defined in component_ids.sh
+# In case new ones are needed to be created, there's guidance below and in component_ids.sh.
+
+# ID of a new base that is intended to contain one large table.
+# The table will be filled using this python script.
+# If the ID is not in the environment, it is possible to create a new base
+# via the Airtable UI, and get the base ID from the URL structure.
+# (https://support.airtable.com/docs/finding-airtable-ids)
+large_table_base_id = os.environ["LARGE_TABLE_BASE_ID"]
+
+# ID of the one table inside the base "large_table_base".
+# The table is intended to be large, and will be filled using this python script.
+# If the ID is not in the environment, it is possible to create a new table
+# via the Airtable UI, and get the table ID from the URL structure.
+# (https://support.airtable.com/docs/finding-airtable-ids)
+large_table_table_id = os.environ["LARGE_TABLE_TABLE_ID"]
+
+# ID of a base that is intended to contain lots of tables.
+# large_base_base_id = os.environ["LARGE_BASE_BASE_ID"]
+# Creating tables is not yet supported in pyairtable. Try Airtable Web API instead:
+# https://airtable.com/developers/web/api/create-base"
+
+
+def create_n_bases(api, number_of_bases):
+    raise NotImplementedError(
+        "Creating bases is not yet supported in pyairtable. \
+                              Try Airtable Web API instead: \
+                              https://airtable.com/developers/web/api/create-base",
+    )
+    # if len(pyair.metadata.get_api_bases(api)["bases"])>99:
+    #     logger.warning("Airtable Org already has a high number of bases. \
+    #                Aborting creation of new bases to avoid duplication and bloating.")
+    #     return
+
+    number_of_bases
+
+
+def create_n_tables(base, number_of_tables):
+    raise NotImplementedError(
+        "Creating tables is not yet supported in pyairtable. \
+                              Try Airtable Web API instead: \
+                              https://airtable.com/developers/web/api/create-table",
+    )
+    # if len(pyair.metadata.get_base_schema(base)["tables"])>99:
+    #     logger.warning("Base already has a high number of tables. \
+    #               Aborting creation of new tables to avoid duplication and bloating.")
+    #     return
+
+
+def create_n_records(table, number_of_records):
+    logger.warning(
+        "Fetching table records to count, before creation of new records.\
+                   This should take around 1 second per 415 records.",
+    )
+    if len(table.all()) > SCALE_TEST_NUMBER_OF_RECORDS - 1:
+        logger.warning(
+            "Table already has a high number of records. \
+                Aborting creation of new records to avoid duplication and bloating.",
+        )
+        return
+
+    records = [{"Name": f"My Name is {i}"} for i in range(number_of_records)]
+    table.batch_create(records)
+
+
+if __name__ == "__main__":
+    api = Api(token)
+    large_table = api.table(large_table_base_id, large_table_table_id)
+    logger.info("Creating records, this should take about 1 second per 40 records.")
+    create_n_records(large_table, SCALE_TEST_NUMBER_OF_RECORDS)

scripts/airtable-test-helpers/create_scale_test_components.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+
+# This scripts creates a large number of tables inside an Airtable base.
+
+# shellcheck disable=SC2001,SC1091
+source ./scripts/airtable-test-helpers/component_ids.sh
+
+base_data='{"description": "Table-X of the test tables for the test LARGE_BASE.", "fields": [{"description": "Name of the row","name": "Name","type": "singleLineText"}],"name": "LARGE_BASE_TABLE_X"}'
+for ((i = 1; i <= 100; i++)); do
+  item="$(echo "$base_data" | sed "s/X/$i/g")"
+  curl -X POST "https://api.airtable.com/v0/meta/bases/$LARGE_BASE_BASE_ID/tables" \
+    -H "Authorization: Bearer $AIRTABLE_ACCESS_TOKEN_WRITE2" \
+    -H "Content-Type: application/json" \
+    --data "$item"
+done

scripts/airtable-test-helpers/print_num_rows_df.py

@@ -0,0 +1,28 @@
+import argparse
+import json
+from io import StringIO
+
+import pandas as pd
+
+
+def number_of_rows(file_path):
+    with open(file_path) as file:
+        data = json.load(file)
+        df = pd.read_csv(StringIO(data[0]["text"]))
+    return len(df)
+
+
+if __name__ == "__main__":
+    parser = argparse.ArgumentParser(
+        description="Read Unstructured Ingest output file and print the number of rows",
+    )
+
+    parser.add_argument(
+        "--structured-output-file-path",
+        help="Path to Unstructured Ingest output file",
+    )
+
+    args = parser.parse_args()
+
+    output_path = args.structured_output_file_path
+    print(number_of_rows(output_path))