wehub-resource-sync
69 lines
2.5 KiB
Python
69 lines
2.5 KiB
Python
|
|
"""This example demonstrates how to parse individual documents into memory
|
|
-- Does not require the use of database
|
|
"""
|
|
|
|
import os
|
|
from llmware.parsers import Parser
|
|
from llmware.setup import Setup
|
|
|
|
|
|
def parsing_files_into_memory(folder_name):
|
|
|
|
print(f"\nExample - Parsing Documents into Memory (no DB required)")
|
|
|
|
# Load the llmware sample files
|
|
print (f"\nstep 1 - loading the llmware sample files")
|
|
sample_files_path = Setup().load_sample_files()
|
|
print (f"step 2- llmware sample files saved locally at: {sample_files_path}")
|
|
|
|
# Parse individual documents. The output will be a list of blocks (dicts with metadata)
|
|
ingestion_file_path = os.path.join(sample_files_path,folder_name)
|
|
|
|
files = os.listdir(ingestion_file_path)
|
|
|
|
print(f"\t\t--total of {len(files)} documents found in folder path - will parse each one.")
|
|
|
|
for i, doc in enumerate(files):
|
|
|
|
parser_output = Parser().parse_one(ingestion_file_path,doc,save_history=False)
|
|
|
|
if parser_output:
|
|
print(f"\nupdate: parsed document - {i} - {doc}")
|
|
print(f"update: extracted {len(parser_output)} blocks of information - showing first 10 blocks for display:")
|
|
|
|
if len(parser_output) < 10:
|
|
sample_display_len = len(parser_output)
|
|
else:
|
|
sample_display_len = 10
|
|
|
|
parser_output = parser_output[0:sample_display_len]
|
|
|
|
for j, blocks in enumerate(parser_output):
|
|
text = blocks["text"]
|
|
if len(text) > 100:
|
|
text = text[0:100] + " ... "
|
|
print(f"update: block - {blocks['block_ID']} - {text}")
|
|
else:
|
|
print(f"\nupdate: did not find any content for file - {doc}")
|
|
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
|
|
# note on sample documents - downloaded by Setup()
|
|
# UN-Resolutions-500 is 500 pdf documents
|
|
# Invoices is 40 pdf invoice samples
|
|
# Agreements is ~15 contract documents
|
|
# AgreementsLarge is ~80 contract documents
|
|
# FinDocs is ~15 financial annual reports and earnings
|
|
# SmallLibrary is a mix of ~10 pdf and office documents
|
|
|
|
# this is a list of document folders that will be pulled down by calling Setup()
|
|
sample_folders = ["Agreements", "Invoices", "UN-Resolutions-500", "SmallLibrary", "FinDocs", "AgreementsLarge"]
|
|
|
|
library_name = "parsing_test_lib_0"
|
|
selected_folder = sample_folders[0]
|
|
output = parsing_files_into_memory (selected_folder)
|