wehub-resource-sync
86 lines
4.2 KiB
Python
86 lines
4.2 KiB
Python
|
|
""" Starting with llmware 0.2.7, new configuration options are exposed in the PDF parser, with many more to come.
|
|
This should not result in any breaking changes in existing code, but exposes new configuration options.
|
|
|
|
Note: default encoding is now 'utf-8' which could result in changes from previous default 'ascii' -> if you
|
|
experience any issues, you can explicitly set the encoding = 'ascii' ... """
|
|
|
|
from llmware.library import Library
|
|
|
|
fp = "/path/to/my/pdf/files"
|
|
|
|
lib = Library().create_new_library("my_library")
|
|
|
|
# standard call to 'ingest' files into a library (implicitly calls Parser and manages the details)
|
|
lib.add_files(input_folder_path=fp)
|
|
|
|
# new configuration options in Parser class, exposed in .add_files method for convenience, although it can
|
|
# always be accessed by direct construction of a Parser
|
|
|
|
# CHUNK SIZING - implemented in PDF parser (and will be supported fully in Office parser coming soon)
|
|
# Measured in string len characters, e.g., 400 characters as target text chunk size, with max of 600 characters
|
|
|
|
# -- there are 3 chunking strategies supported:
|
|
# -- smart_chunking = 0 -> will stop at the target chunk size (or as close as possible) - breaks words
|
|
# -- smart_chunking = 1 -> will stop at the first white space after the target chunk size - preserves words
|
|
# -- smart_chunking = 2 -> will look for a natural break, either a "." or "\r" or "\n", up to the max size
|
|
# -- note: this is applied on a page-by-page basis, so if there are 1800 characters on a page, then in this example,
|
|
# it would result in 4 text chunks of ~400 characters, and a final chunk of the remainder of ~200 characters
|
|
|
|
lib.add_files(input_folder_path=fp, chunk_size=400, max_chunk_size=600, smart_chunking=0)
|
|
|
|
# CAPTURE PARAMETERS
|
|
|
|
# -- ability to turn on/off the capture of other features in the parser, e.g, whether to capture images, tables,
|
|
# and formatted header text (e.g., BOLD, ITALICS and Large Font)
|
|
|
|
# -- if you are only interested in text, then you can turn off these features for faster performance
|
|
|
|
lib.add_files(input_folder_path=fp, get_images=False, get_tables=False, get_header_text=False)
|
|
|
|
# ENCODING
|
|
|
|
# -- encoding = "utf-8" - new default - supports Western European characters, and many characters across the
|
|
# wider Unicode set of code points - note: not full support yet for Asian languages and characters
|
|
|
|
# -- backup option = "ascii" - if english only document, then this will often times result in a slightly cleaner
|
|
# output as it will restrict the output to ASCII 7-bit range (1-127)
|
|
|
|
# -- backup option (not recommended, but available) = "latin-1" encoding - will attempt to save the 8-bit
|
|
# character directly into the database with UTF-8 bit mapping - which can create downstream validation problems
|
|
# but available if needed in a special case
|
|
|
|
lib.add_files(input_folder_path=fp, encoding="utf-8")
|
|
|
|
# STRIP HEADERS
|
|
|
|
# -- some documents prepared by a 3rd party service will insert a repeating header at the top of each page
|
|
# -- if strip_header == True, then the parser will skip the first text box found on each page in an attempt to
|
|
# to remove repeating common header (very useful in some cases- and worth trying)
|
|
|
|
lib.add_files(input_folder_path=fp, strip_header=True)
|
|
|
|
# COPY FILES TO LIBRARY
|
|
|
|
# -- by default, input files are collated to different parsers, and then at the end of the process, copied to a
|
|
# library file structure for convenient future access by the library and for downstream retrieval applications
|
|
# that query the library, and then want to provide direct access to the files.
|
|
|
|
# -- if you do not want to duplicate files, then set copy_files_to_library = False
|
|
|
|
lib.add_files(input_folder_path=fp, copy_files_to_library=False)
|
|
|
|
# VERBOSE OUTPUT
|
|
|
|
# -- four modes of output to screen
|
|
# -- verbose_level == 0 -> suppresses virtually all output, except major errors/problems
|
|
# -- verbose_level == 1 -> displays 1st ten pages of document text in text chunks as parsing
|
|
# -- verbose_level == 2 -> displays file name being parsed only
|
|
# -- verbose_level == 3 -> deep debugging mode (not recommended) - useful for llmware dev team in tracing errors
|
|
|
|
lib.add_files(input_folder_path=fp, verbose_level=2)
|
|
|
|
|
|
|
|
|