chore: import upstream snapshot with attribution

tutorials/getting_started/README.md

@@ -0,0 +1,45 @@
+ 🚀 Getting Set up - Key Items  🚀  
+===============
+
+**How to get started with LLMWare?**    
+
+In this repository, we have small code snippets with a lot of annotation that describe key setup items to get started quickly and successfully with LLMWare:
+
+1.  [**Configuring a DB**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/configure_db.py) - for a fast start, you don't have to do anything except set the .active_db parameter to "sqlite" and all libraries will be written on a local embedded sqlite instance with no installation required.  
+
+
+    from llmware.configs import LLMWareConfig
+    LLMWareConfig().set_active_db("sqlite")
+    
+
+   After writing content into your first library, you will find the sqlite db at the path defined here:  
+    
+    from llmware.configs import SQLiteConfig
+    SQLiteConfig().get_config("sqlite_db_folder_path")
+
+
+2.  [**Loading Sample Files**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/loading_sample_files.py) - for an overview of the sample files available and how to pull down (integrated into many examples too.)  
+
+
+    from llmware.setup import Setup
+    Setup().load_sample_files()
+
+
+3.  [**Using the Model Catalog**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/using_the_model_catalog.py) - this is the primary model discovery and loading class for LLMWare.  
+
+
+    from llmware.models import ModelCatalog
+    all_models = ModelCatalog().list_all_models()
+    for model in all_models: print("model: ", model)
+
+
+4.  [**Working with Libraries**](https://www.github.com/llmware-ai/llmware/tree/main/examples/Getting_Started/working_with_libraries.py) - Libraries are the main organizing construct for text collections in LLMWare.  
+
+
+As you are getting started, we would also recommend that you check out the [**Fast Start**](https://www.github.com/llmware-ai/llmware/tree/main/fast_start) examples and videos as well!  
+
+
+
+### **Let's get started!  🚀**
+
+

tutorials/getting_started/configure_db.py

@@ -0,0 +1,45 @@
+
+""" This example shows how to select and set the collection database. The collection DB is used as the primary
+source of storing text chunks from parsing runs, and organizing into Libraries.  LLMWare supports three
+collection databases:  MongoDB, Postgres, and SQLite.
+
+    For a fast no-install start, we would recommend SQLite.
+
+    If you prefer SQL and massive scalability, we would recommend Postgres.
+
+    If you prefer no-SQL and combo of scale and flexibility, we would recommend MongoDB.
+
+    The LLMWare functions, methods and interfaces to each of the DBs are exactly the same with a
+    high-level abstraction interface provided within LLMWare CollectionRetrieval and CollectionWriter classes
+    that handle the implementation-specific details of writing and retrieving from each of these data sources.
+
+    While semantic retrieval requires the use of a separate vector database and creating an embedding for the
+    content in the Library, once the library is created, text search retrieval is available automatically so text
+    Queries can be run immediately after Parsing.
+
+"""
+
+from llmware.configs import LLMWareConfig
+
+
+def set_collection_database():
+
+    #   check the current active db
+    active_db = LLMWareConfig().get_active_db()
+
+    print("update: current 'active' collection database - ", active_db)
+
+    #   supported db list
+    supported_db = LLMWareConfig().get_supported_collection_db()
+
+    print("update: supported collection databases - ", supported_db)
+
+    #   set the current active db -> once set, any calls to library / parsing will write to this db
+    LLMWareConfig.set_active_db("sqlite")
+
+    #   change anytime -> going forward, any new libraries will be written to the new db, but existing libraries
+    #   remain on the previous db
+    LLMWareConfig.set_active_db("postgres")
+
+    return 0
+

tutorials/getting_started/loading_sample_files.py

@@ -0,0 +1,94 @@
+
+""" To enable rapid testing and prototyping, LLMWare provides a range of sample files that can be accessed
+through the Setup class, and specific methods that will pull the files from a non-restricted llmware AWS S3 bucket,
+and download the files locally in the /llmware_data/sample_files path.
+
+    Sample files are created from public domain sources, and often developed originally by LLMWare.   They are
+provided for convenience in testing.   As we find or develop a good testing set, we will generally try to make it
+available so everyone can use - as a result, the sample files collection is evolving and growing all the time!
+
+    If you have an older version, and would like to get the latest, then you can set the over_write=True option.
+
+"""
+
+import os
+from llmware.setup import Setup
+
+
+def get_llmware_sample_files():
+
+    print (f"\n > Loading the llmware Sample Files...")
+
+    #   this call to Setup() will pull the sample files from a public S3 repo
+    #   the files will be placed in folders locally at the sample_files_path
+
+    sample_files_path = Setup().load_sample_files(over_write=False)
+
+    print(f"> Sample Files Path - {sample_files_path}")
+
+    print(f"> Sample Folders - {os.listdir(sample_files_path)}")
+
+    #   Current Sample Files:
+    #
+    #   AgreementsLarge     =   ~80 sample contracts
+    #   Agreements          =   ~15 sample employment agreements
+    #   UN-Resolutions-500  =   500 United Nations Resolutions (~2 years) - public repo - PDF files
+    #   Invoices            =   ~40 invoice sample documents
+    #   FinDocs             =   ~15 financial annual reports, earnings and 10Ks
+    #   AWS-Transcribe      =   ~5 AWS-transcribe JSON files
+    #   SmallLibrary        =   ~10 mixed document types for quick testing
+    #   Images              =   ~3 images for OCR processing
+
+    #   Note: these files will be updated from time-to-time - if you want to pull fresh new files
+    #   sample_files_path = Setup().load_sample_files(over_write=True)
+
+    #   These files were prepared by LLMWare from public domain materials or invented bespoke as examples
+    #   If you have any concerns about PII or the suitability or any material, please let us know
+    #   We reserve the right to withdraw documents at any time
+
+    return 0
+
+
+def get_llmware_voice_sample_files():
+
+    print (f"\n > Loading the llmware Sample Voice Files...")
+
+    #   check out the examples:  Models/using-whisper-cpp-sample-files.py and
+    #                            Use_Cases/parsing_great_speeches.py
+
+    #   this call to Setup() will pull the sample files from a public S3 repo
+    #   the files will be placed in folders locally at the sample voice files path
+
+    sample_files_path = Setup().load_voice_sample_files(over_write=False,small_only=False)
+
+    print(f"> Sample Voice Files Path - {sample_files_path}")
+
+    print(f"> Sample Voice Folders - {os.listdir(sample_files_path)}")
+
+    # examples - "famous_quotes" | "greatest_speeches" | "youtube_demos" | "earnings_calls"
+
+    #   -- famous_quotes - approximately 20 small .wav files with clips from old movies and speeches
+    #   -- greatest_speeches - approximately 60 famous historical speeches in english
+    #   -- youtube_videos - wav files of ~3 llmware youtube videos
+    #   -- earnings_calls - wav files of ~4 public company earnings calls (gathered from public investor relations)
+
+    # These sample files are hosted in a non-restricted AWS S3 bucket, and downloaded via the Setup method
+    # `load_sample_voice_files`.   There are two options:
+
+    #    --  small_only = True:      only pulls the 'famous_quotes' samples
+    #    --  small_only = False:     pulls all of the samples    (requires ~1.9 GB in total)
+
+    # Please note that all of these samples have been pulled from open public domain sources, including the
+    # Internet Archives, e.g., https://archive.org.  These sample files are being provided solely for the purpose of
+    # testing the code scripts below.   Please do not use them for any other purpose.
+
+    return 0
+
+
+if __name__ == "__main__":
+
+    get_llmware_sample_files()
+
+    get_llmware_voice_sample_files()
+
+

tutorials/getting_started/using_the_model_catalog.py

@@ -0,0 +1,78 @@
+
+""" This example shows how to get started using the ModelCatalog to instantiate models and start running inferences.
+
+    A core design principle of LLMWare is that all models should be accessible in the same way, with the
+    underlying configuration details handled at lower levels of the implementation.   Our aspiration is that you
+    should be able to substitute models in any pipeline/workflow at any point with minimal, if any, code change -
+    and to be able to deploy heterogeneous combinations of local, private network and API-based models,
+    with mix-qnd-match and full 'swqp-ability' at any time.
+
+    While LLMWare supports OpenAI and other API-based models, we are "open-source-first" in our view and
+    support of models, and all of our examples and classes/methods are optimized first to work with small, specialized
+    open source models.   Our view is that essentially anything that can be done with large API models can be
+    replicated with smaller fine-tuned models and well-designed data pipelines and workflows.
+
+    We provide over 50+ LLMWare models that are all tested and designed for easy integration
+    into LLMWare pipeline and workflows, but we are 100% committed to providing an open ecosystem and supporting
+    the best in open source, including leading embedding models from Jina and Nomic, Llama-2, Llama-3, Mistral,
+    Phi-3, Yi, DeciLM, StableLM, OpenHermes, Tiny Llama, RedPajama, Pythia, Sheared LLama, Huggingface Zephyr,
+    SentenceTransformers and quantizations from The Bloke and Bartowski.  We also provide a full integration into GGUF,
+    LLama.CPP and Whisper.CPP.
+
+    It is easy to extend the ModelCatalog to include any of your favorite models from
+    HuggingFace, Ollama, LMStudio, LLama.CPP, or llama-cpp-python.
+
+    For open source models, we generally support both PyTorch (Huggingface) based models, as well as GGUF (LLama.CPP)
+    quantized models.  For most use cases, we find that GGUF quantized versions are faster, take less memory
+    and are generally as accurate, so we use GGUF-based models, aka "tools", in many of our examples.
+
+    """
+
+from llmware.models import ModelCatalog
+
+all_models = ModelCatalog().list_all_models()
+
+#   ~133 models in the catalog as of May 4, 2024 - and growing all the time!
+print("\nAll Models")
+
+for i, model in enumerate(all_models):
+    print("models: ", i, model)
+
+#   ~30 embedding models - including Nomic, Jina, leading sentence transformers, llmware industry-bert models
+embedding_models = ModelCatalog().list_embedding_models()
+
+"""
+print("\nEmbedding Models")
+for i, model in enumerate(embedding_models):
+    print("embedding models: ", i, model)
+"""
+
+#   ~92 open source models
+open_source_models = ModelCatalog().list_open_source_models()
+
+"""
+print("\nOpen Source Models")
+for i, model in enumerate(open_source_models):
+    print("open source models: ", i, model)
+"""
+
+#   Inference is Easy - same two lines every time
+#   1.  load_model - use the model_name or display_name, and it will be looked up and instantiated
+#   2.  inference -  all models support an inference method - call it to run a basic inference
+
+model_name = "phi-3-gguf"
+model = ModelCatalog().load_model(model_name)
+response = model.inference("I am going to Paris.  What should I see?")
+
+print(f"\ntest inference - {model_name} - response: ", response)
+
+#   Check out other examples in Models, Prompts, SLIM-Agents Embeddings, and Use_Cases
+#   -- configuration in loading model - temperature, sample, max_output
+#   -- add models to the Catalog for easy invocation
+#   -- integrating sources and building RAG workflows in Prompts
+#   -- fact-checking and post-processing
+#   -- using function-calls on small specialized models
+#   -- agent based workflows
+#   -- installing embeddings on a library collection
+
+

tutorials/getting_started/welcome_example.py

@@ -0,0 +1,72 @@
+
+""" This 'welcome_example' can serve as a quick 'hello world' test to verify that LLMWare has been installed
+and that local model inference serving over GGUF is available and working as expected.   This example will
+pull down two models and run a quick 'Welcome' for a new user.  """
+
+
+from llmware.models import ModelCatalog
+from llmware.configs import LLMWareConfig
+
+#   optional - adds a dash of color to the console output
+try:
+    from colorama import Fore
+    BLUE = Fore.BLUE
+    RESET = Fore.RESET
+except:
+    BLUE = ""
+    RESET = ""
+
+print(f"\n{BLUE}Welcome to LLMWare - Test Script{RESET}")
+
+#   run first inference
+print(f"\nLoading {BLUE}bling-phi-3-gguf model{RESET} for First Inference: may take a minute to download the first time")
+print(f"Model will be cached at the local path: {BLUE}{LLMWareConfig().get_model_repo_path()}{RESET}")
+
+try:
+
+    #   loads the model from the model catalog
+    model = ModelCatalog().load_model("bling-phi-3-gguf", temperature=0.0, sample=False)
+
+    prompt = ("When this script is loaded, we should greet the person by saying, 'Welcome to LLMWare.'"
+            "\nThis script has been loaded - what should we say?")
+
+    print(f"\nprompt: {prompt}")
+
+    #   executes inference
+    response = model.inference(prompt)
+
+    print("\nmodel response: ", response)
+
+except:
+    print("\nWe are sorry but something has gone wrong with the installation and/or download of the file, and it "
+          "did not start correctly.   Please check the documentation.\nMost common sources of this error: "
+          "\n1.  Supported platforms - Mac M1/M2/M3, Linux x86, Windows"
+          "\n2.  Model did not download correctly.  Try again and confirm that model instantiated in local folder path."
+          "\n3.  Update/pull from the latest repo and/or the latest pip install version.")
+
+
+#   run second inference
+print(f"\nLoading {BLUE}slim-sentiment-tool{RESET}")
+
+try:
+
+    #   load the model from the model catalog
+    model = ModelCatalog().load_model("slim-sentiment-tool", temperature=0.0, sample=False)
+
+    user = "We are very happy and excited to get started with LLMWare."
+    print(f"\nuser: {user}")
+
+    #   execute a 'sentiment' function call on the model
+    response = model.function_call(user, function="classify", params=["sentiment"],get_logits=False)
+
+    print("\nclassifying the sentiment: ", response)
+
+except:
+    print("We are sorry but something has gone wrong with the installation and/or download of the file, and it "
+          "did not start correctly.   Please check the documentation.\nMost common sources of this error: "
+          "\n1.  Supported platforms - Mac M1/M2/M3, Linux x86, Windows"
+          "\n2.  Model did not download correctly.  Try again and confirm that model instantiated in local folder path."
+          "\n3.  Update/pull from the latest repo and/or the latest pip install version.")
+
+
+print(f"\n\nPlease review /examples for other examples.\n\n{BLUE}Welcome to the LLMWare community.{RESET}")

tutorials/getting_started/working_with_libraries.py

@@ -0,0 +1,102 @@
+
+""" This example shows the basics of working with Libraries.  Library is the main organizing construct for
+unstructured information in LLMWare.   Users can create one large library with all types of different content, or
+can create multiple libraries with each library comprising a specific logical collection of information on a
+particular subject matter, project/case/deal, or even different accounts/users/departments.
+
+    Each Library consists of the following components:
+
+    1. Collection on a Database - this is the core of the Library, and is created through parsing of documents, which
+    are then automatically chunked and indexed in a text collection database.  This is the basis for retrieval,
+    and the collection that will be used as the basis for tracking any number of vector embeddings that can be
+    attached to a library collection.
+
+    2. File archives - found in the llmware_data path, within Accounts, there is a folder structure for each Library.
+    All file-based artifacts for the Library are organized in these folders, including copies of all files added in
+    the library (very useful for retrieval-based applications), images extracted and indexed from the source
+    documents, as well as derived artifacts such as nlp and knowledge graph and datasets.
+
+    3. Library Catalog - each Library is registered in the LibraryCatalog table, with a unique library_card that has
+    the key attributes and statistics of the Library.
+
+    When a Library object is passed to the Parser, the parser will automatically route all information into the
+    Library structure.
+
+    The Library also exposes convenience methods to easily install embeddings on a library, including tracking of
+    incremental progress.
+
+    To parse into a Library, there is the very useful convenience methods, "add_files" which will invoke the Parser,
+    collate and route the files within a selected folder path, check for duplicate files, execute the parsing,
+    text chunking and insertion into the database, and update all of the Library state automatically.
+
+    Libraries are the main index constructs that are used in executing a Query.   Pass the library object when
+    constructing the Query object, and then all retrievals (text, semantic and hybrid) will be executed against
+    the content in that Library only.
+
+"""
+
+import json
+import os
+from llmware.configs import LLMWareConfig
+from llmware.library import Library, LibraryCatalog
+from llmware.setup import Setup
+
+
+def core_library_functions(library_name):
+
+    # Create a library
+    print (f"\n > Creating library {library_name}...")
+
+    library = Library().create_new_library(library_name)
+
+    # Load an existing library. This not required after library creation and is only shown here for reference
+    library = Library().load_library(library_name)
+
+    # The LibraryCatalog is used to query all libraries
+    print (f"\n > All libraries")
+    for library_card in LibraryCatalog().all_library_cards():
+        lib_name = library_card["library_name"]
+        docs = library_card["documents"]
+        print (f"  - {lib_name} ({docs} documents)")
+
+    # Add a few files to our library
+    print (f"\n > Adding some files to {library_name}")
+    sample_files_path = Setup().load_sample_files()
+    library.add_files(os.path.join(sample_files_path,"SmallLibrary"))
+
+    # View the library card to confirm document, block and other counts
+    print (f"\n > Library Card")
+    library_card = library.get_library_card()
+    library_card["_id"] = str(library_card["_id"]) # The _id needs to be converted to a str before printing
+    print (json.dumps(library_card, indent=2))
+ 
+    # Library Export to JSON
+    print (f"\n > Exporting library to jsonl file...")
+    temp_export_dir = LLMWareConfig().get_tmp_path()
+    json_export_path = library.export_library_to_jsonl_file(temp_export_dir, "lib_export")
+    print (f" - library exported to {json_export_path}")
+    
+    # Library export to txt file
+    print (f"\n > Exporting library to text file...")
+    text_export_path = library.export_library_to_txt_file(temp_export_dir, "lib_export")
+    print (f" - library exported to {text_export_path}")
+
+    # check out the images extracted
+    print (f"\n images extracted and saved at local path - {library.image_path}")
+
+    # Delete the library
+    # print (f"\n > Deleting the library...")
+    # note: for safety: set the confirm_delete=True when/if you want to delete
+    library.delete_library(confirm_delete=False)
+
+
+if __name__ == "__main__":
+
+    #   set the database to be used to store Library text collection
+    #   3 supported options:  MongoDB, Postgres, SQLite
+    #   MongoDB and Postgres require separate install, while SQLite is no-install/setup
+
+    LLMWareConfig().set_active_db("sqlite")
+
+    core_library_functions("library_tests2")
+